From 2f736a179a64a63feb239ee8f3200856a05c49ba Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 18 Apr 2024 18:33:20 -0700 Subject: [PATCH 01/88] Release Process: --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3842c6793..da2d60b2f 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ under the License. org.apache.datasketches datasketches-java - 6.0.0-SNAPSHOT + 6.1.0-SNAPSHOT jar ${project.artifactId} From 52d29de27c1abb00b15eef4956bedd3fb4ab1912 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Sun, 21 Apr 2024 18:43:02 -0700 Subject: [PATCH 02/88] Improved javadocs for main code. --- .../cpc/CompressionCharacterization.java | 15 ++++++ .../filters/bloomfilter/BloomFilter.java | 18 ++++++- .../filters/bloomfilter/DirectBitArrayR.java | 3 ++ .../apache/datasketches/hll/TgtHllType.java | 19 ++++++- .../datasketches/kll/KllItemsSketch.java | 4 ++ .../kll/KllItemsSketchIterator.java | 1 + .../apache/datasketches/kll/KllSketch.java | 45 ++++++++++++++-- .../datasketches/partitions/BoundsRule.java | 6 +++ .../datasketches/partitions/Partitioner.java | 21 ++++++++ .../partitions/SketchFillRequest.java | 3 +- .../DoublesSortedViewIterator.java | 6 +++ .../FloatsSortedViewIterator.java | 6 +++ .../GenericPartitionBoundaries.java | 11 ++++ .../GenericSortedViewIterator.java | 8 ++- .../quantilescommon/IncludeMinMax.java | 53 +++++++++++++++++++ .../ItemsSketchSortedView.java | 13 +++-- .../quantilescommon/PartitioningFeature.java | 1 + .../quantilescommon/QuantilesAPI.java | 1 + .../quantilescommon/QuantilesUtil.java | 8 +++ .../sampling/EbppsItemsSketch.java | 2 +- .../datasketches/tdigest/TDigestDouble.java | 1 + .../apache/datasketches/theta/BitPacking.java | 25 +++++++-- .../datasketches/theta/CompactSketch.java | 4 ++ .../thetacommon/SetOperationCornerCases.java | 3 ++ .../tuple/SerializerDeserializer.java | 1 + .../org/apache/datasketches/tuple/Util.java | 7 +++ 26 files changed, 270 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/apache/datasketches/cpc/CompressionCharacterization.java b/src/main/java/org/apache/datasketches/cpc/CompressionCharacterization.java index 962b2882c..4f0a93352 100644 --- a/src/main/java/org/apache/datasketches/cpc/CompressionCharacterization.java +++ b/src/main/java/org/apache/datasketches/cpc/CompressionCharacterization.java @@ -68,6 +68,18 @@ public class CompressionCharacterization { private CompressedState[] compressedStates2; private CpcSketch[] unCompressedSketches; + /** + * Only used in test. + * @param lgMinK min lgK + * @param lgMaxK max lgK + * @param lgMinT min lgTrials + * @param lgMaxT max lgTrials + * @param lgMulK lg multiple + * @param uPPO unique axis Points Per Octave + * @param incLgK increment lgK + * @param pS PrintStream + * @param pW PrintWriter + */ @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "This is OK here") public CompressionCharacterization( final int lgMinK, @@ -91,6 +103,9 @@ public CompressionCharacterization( assembleFormats(); } + /** + * Only used in test + */ public void start() { printf(hfmt, (Object[]) hStrArr); //print header doRangeOfLgK(); diff --git a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java index 0c957ce92..3ea73b9bd 100644 --- a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java +++ b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java @@ -54,10 +54,12 @@ * false positive probability.

* *

This implementation uses xxHash64 and follows the approach in Kirsch and Mitzenmacher, - * "Less Hashing, Same Performance: Building a Better Bloom Filter," Wiley Interscience, 2008, - * pp. 187-218.

+ * "Less Hashing, Same Performance: Building a Better Bloom Filter," Wiley Interscience, 2008, pp. 187-218.

*/ public final class BloomFilter { + /** + * The maximum size of a bloom filter in bits. + */ public static final long MAX_SIZE_BITS = (Integer.MAX_VALUE - Family.BLOOMFILTER.getMaxPreLongs()) * (long) Long.SIZE; private static final int SER_VER = 1; private static final int EMPTY_FLAG_MASK = 4; @@ -133,11 +135,23 @@ public static BloomFilter heapify(final Memory mem) { return internalHeapifyOrWrap((WritableMemory) mem, false, false); } + /** + * Wraps the given Memory into this filter class. The class itself only contains a few metadata items and holds + * a reference to the Memory object, which contains all the data. + * @param mem the given Memory object + * @return the wrapping BloomFilter class. + */ public static BloomFilter wrap(final Memory mem) { // casting to writable, but tracking that the object is read-only return internalHeapifyOrWrap((WritableMemory) mem, true, false); } + /** + * Wraps the given WritableMemory into this filter class. The class itself only contains a few metadata items and holds + * a reference to the Memory object, which contains all the data. + * @param wmem the given WritableMemory object + * @return the wrapping BloomFilter class. + */ public static BloomFilter writableWrap(final WritableMemory wmem) { return internalHeapifyOrWrap(wmem, true, true); } diff --git a/src/main/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayR.java b/src/main/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayR.java index 19c495af6..8acc36be2 100644 --- a/src/main/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayR.java +++ b/src/main/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayR.java @@ -24,6 +24,9 @@ import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; +/** + * This class can maintain the BitArray object off-heap. + */ public class DirectBitArrayR extends BitArray { final static protected long NUM_BITS_OFFSET = Long.BYTES; final static protected long DATA_OFFSET = 2L * Long.BYTES; diff --git a/src/main/java/org/apache/datasketches/hll/TgtHllType.java b/src/main/java/org/apache/datasketches/hll/TgtHllType.java index a0ee79a45..a5dc395ce 100644 --- a/src/main/java/org/apache/datasketches/hll/TgtHllType.java +++ b/src/main/java/org/apache/datasketches/hll/TgtHllType.java @@ -50,10 +50,27 @@ * * @author Lee Rhodes */ -public enum TgtHllType { HLL_4, HLL_6, HLL_8; +public enum TgtHllType { + /** + * An HLL sketch with a bin size of 4 bits + */ + HLL_4, + /** + * An HLL sketch with a bin size of 6 bits + */ + HLL_6, + /** + * An Hll Sketch with a bin size of 8 bits + */ + HLL_8; private static final TgtHllType values[] = values(); + /** + * Convert the typeId to the enum type + * @param typeId the given typeId + * @return the enum type + */ public static final TgtHllType fromOrdinal(final int typeId) { return values[typeId]; } diff --git a/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java b/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java index 392da0673..6fb9772fb 100644 --- a/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java @@ -290,6 +290,10 @@ public void reset() { itemsSV = null; } + /** + * Export the current sketch as a compact byte array. + * @return the current sketch as a compact byte array. + */ public byte[] toByteArray() { return KllHelper.toByteArray(this, false); } diff --git a/src/main/java/org/apache/datasketches/kll/KllItemsSketchIterator.java b/src/main/java/org/apache/datasketches/kll/KllItemsSketchIterator.java index 3a0a8da0f..02bda7a20 100644 --- a/src/main/java/org/apache/datasketches/kll/KllItemsSketchIterator.java +++ b/src/main/java/org/apache/datasketches/kll/KllItemsSketchIterator.java @@ -23,6 +23,7 @@ /** * Iterator over KllItemsSketch. The order is not defined. + * @param the item class type */ public final class KllItemsSketchIterator extends KllSketchIterator implements QuantilesGenericSketchIterator { private final Object[] quantiles; diff --git a/src/main/java/org/apache/datasketches/kll/KllSketch.java b/src/main/java/org/apache/datasketches/kll/KllSketch.java index 4ce15aba9..c398ed8ce 100644 --- a/src/main/java/org/apache/datasketches/kll/KllSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllSketch.java @@ -218,6 +218,10 @@ public boolean hasMemory() { return (wmem != null); } + /** + * Returns true if this sketch is in a Compact Memory Format. + * @return true if this sketch is in a Compact Memory Format. + */ public boolean isCompactMemoryFormat() { return hasMemory() && sketchStructure != UPDATABLE; } @@ -488,9 +492,18 @@ final void setLevelsArrayAt(final int index, final int idxVal) { * Used to define the variable type of the current instance of this class. */ public enum SketchType { - DOUBLES_SKETCH(Double.BYTES, "DoublesSketch"), - FLOATS_SKETCH(Float.BYTES, "FloatsSketch"), - ITEMS_SKETCH(0, "ItemsSketch"); + /** + * KllDoublesSketch + */ + DOUBLES_SKETCH(Double.BYTES, "KllDoublesSketch"), + /** + * KllFloatsSketch + */ + FLOATS_SKETCH(Float.BYTES, "KllFloatsSketch"), + /** + * KllItemsSketch + */ + ITEMS_SKETCH(0, "KllItemsSketch"); private int typeBytes; private String name; @@ -500,8 +513,16 @@ private SketchType(final int typeBytes, final String name) { this.name = name; } + /** + * Gets the item size in bytes. If the item is generic, this returns zero. + * @return the item size in bytes + */ public int getBytes() { return typeBytes; } + /** + * Get the name of the associated sketch + * @return the name of the associated sketch + */ public String getName() { return name; } } @@ -509,9 +530,13 @@ private SketchType(final int typeBytes, final String name) { * Used primarily to define the structure of the serialized sketch. Also used by the Heap Sketch. */ public enum SketchStructure { + /** Compact Empty Structure */ COMPACT_EMPTY(PREAMBLE_INTS_EMPTY_SINGLE, SERIAL_VERSION_EMPTY_FULL), + /** Compact Single Item Structure */ COMPACT_SINGLE(PREAMBLE_INTS_EMPTY_SINGLE, SERIAL_VERSION_SINGLE), + /** Compact Full Preamble Structure */ COMPACT_FULL(PREAMBLE_INTS_FULL, SERIAL_VERSION_EMPTY_FULL), + /** Updatable Preamble Structure */ UPDATABLE(PREAMBLE_INTS_FULL, SERIAL_VERSION_UPDATABLE); //also used by the heap sketch. private int preInts; @@ -522,10 +547,24 @@ private SketchStructure(final int preInts, final int serVer) { this.serVer = serVer; } + /** + * gets the Preamble Integers for this Structure. + * @return the Preamble Integers for this Structure + */ public int getPreInts() { return preInts; } + /** + * gets the Serialization Version for this Structure. + * @return the Serialization Version for this Structure. + */ public int getSerVer() { return serVer; } + /** + * gets the SketchStructure given preInts and serVer. + * @param preInts the given preamble size in integers + * @param serVer the given Serialization Version + * @return the SketchStructure given preInts and serVer. + */ public static SketchStructure getSketchStructure(final int preInts, final int serVer) { final SketchStructure[] ssArr = SketchStructure.values(); for (int i = 0; i < ssArr.length; i++) { diff --git a/src/main/java/org/apache/datasketches/partitions/BoundsRule.java b/src/main/java/org/apache/datasketches/partitions/BoundsRule.java index ecda05e1b..3cd38996b 100644 --- a/src/main/java/org/apache/datasketches/partitions/BoundsRule.java +++ b/src/main/java/org/apache/datasketches/partitions/BoundsRule.java @@ -19,6 +19,10 @@ package org.apache.datasketches.partitions; +/** + * This instructs the user about which of the upper and lower bounds of a partition definition row + * should be included with the returned data. + */ public enum BoundsRule { /** @@ -30,10 +34,12 @@ public enum BoundsRule { * Include only the upper bound but not the lower bound */ INCLUDE_UPPER, + /** * Include only the lower bound but not the upper bound */ INCLUDE_LOWER, + /** * Include none */ diff --git a/src/main/java/org/apache/datasketches/partitions/Partitioner.java b/src/main/java/org/apache/datasketches/partitions/Partitioner.java index 66030fb22..dd69f1ec2 100644 --- a/src/main/java/org/apache/datasketches/partitions/Partitioner.java +++ b/src/main/java/org/apache/datasketches/partitions/Partitioner.java @@ -162,12 +162,22 @@ private void partitionSearch(final ArrayDeque> stack) { /** * Holds data for a Stack element + * @param the item class type */ public static class StackElement { + /** A reference to the relevant GenericPartitionBoundaries class */ public final GenericPartitionBoundaries gpb; + /** The partition index */ public int part; + /** A brief string description of the partition and its hierarchy */ public String levelPartId; + /** + * Constructs this StackElement + * @param gpb the given GenericPartitionBoundarie reference + * @param part The partition index + * @param levelPartId A brief string description of the partition and its hierarchy + */ public StackElement(final GenericPartitionBoundaries gpb, final int part, final String levelPartId) { this.gpb = gpb; this.part = part; @@ -177,15 +187,26 @@ public StackElement(final GenericPartitionBoundaries gpb, final int part, fin /** * Defines a row for List of PartitionBounds. + * @param the item class type */ public static class PartitionBoundsRow { + /** The partition index */ public int part; + /** A brief string description of the partition and its hierarchy */ public String levelPartId; + /** The approximate number of items represented by this partition description row. */ public long approxNumDeltaItems; + /** The BoundsRule for this partition description row. */ public BoundsRule rule; + /** The lower bound value */ public T lowerBound; + /** The upper bound value */ public T upperBound; + /** + * The constructor for the StackElement class. + * @param se the given stack element. + */ public PartitionBoundsRow(final StackElement se) { final GenericPartitionBoundaries gpb = se.gpb; final QuantileSearchCriteria searchCrit = gpb.getSearchCriteria(); diff --git a/src/main/java/org/apache/datasketches/partitions/SketchFillRequest.java b/src/main/java/org/apache/datasketches/partitions/SketchFillRequest.java index d005561d0..76e9d7454 100644 --- a/src/main/java/org/apache/datasketches/partitions/SketchFillRequest.java +++ b/src/main/java/org/apache/datasketches/partitions/SketchFillRequest.java @@ -25,7 +25,8 @@ /** * This is a callback request to the data source to fill a quantiles sketch, * which is returned to the caller. - * + * @param the item class type + * @param the sketch type * @author Lee Rhodes */ public interface SketchFillRequest & PartitioningFeature> { diff --git a/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedViewIterator.java b/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedViewIterator.java index da112dc2e..85bf96b68 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedViewIterator.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedViewIterator.java @@ -25,6 +25,12 @@ public final class DoublesSortedViewIterator extends SortedViewIterator { private final double[] quantiles; + /** + * Constructor. + * @param quantiles the given array of quantiles, which must be ordered. + * @param cumWeights the given array of cumulative weights, which must be ordered, start with the value one, and + * the last value must be equal to N, the total number of items updated to the sketch. + */ public DoublesSortedViewIterator(final double[] quantiles, final long[] cumWeights) { super(cumWeights); this.quantiles = quantiles; //SpotBugs EI_EXPOSE_REP2 suppressed by FindBugsExcludeFilter diff --git a/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedViewIterator.java b/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedViewIterator.java index a40bacef1..09b45da01 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedViewIterator.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedViewIterator.java @@ -25,6 +25,12 @@ public final class FloatsSortedViewIterator extends SortedViewIterator { private final float[] quantiles; + /** + * Constructor. + * @param quantiles the given array of quantiles, which must be ordered. + * @param cumWeights the given array of cumulative weights, which must be ordered, start with the value one, and + * the last value must be equal to N, the total number of items updated to the sketch. + */ public FloatsSortedViewIterator(final float[] quantiles, final long[] cumWeights) { super(cumWeights); this.quantiles = quantiles; //SpotBugs EI_EXPOSE_REP2 suppressed by FindBugsExcludeFilter diff --git a/src/main/java/org/apache/datasketches/quantilescommon/GenericPartitionBoundaries.java b/src/main/java/org/apache/datasketches/quantilescommon/GenericPartitionBoundaries.java index ee53e910f..0eb44f3e7 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/GenericPartitionBoundaries.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/GenericPartitionBoundaries.java @@ -27,6 +27,7 @@ /** * This defines the returned results of the getParitionBoundaries() function and * includes the basic methods needed to construct actual partitions. + * @param the item class type */ public final class GenericPartitionBoundaries { private long totalN; //totalN of source sketch @@ -40,6 +41,16 @@ public final class GenericPartitionBoundaries { private long[] numDeltaItems; //num of items in each partition private int numPartitions; //num of partitions + /** + * Constructor. + * @param totalN the total number of items input to the sketch. + * @param boundaries The quantile boundaries between partitions + * @param natRanks The array of natural Ranks corresponding to the array of boundaries. + * @param normRanks The normalized Ranks corresponding to the array of boundaries. + * @param maxItem the maximum item of the stream. + * @param minItem the minimum item of the stream. + * @param searchCrit the user defined search criteria + */ public GenericPartitionBoundaries( final long totalN, final T[] boundaries, diff --git a/src/main/java/org/apache/datasketches/quantilescommon/GenericSortedViewIterator.java b/src/main/java/org/apache/datasketches/quantilescommon/GenericSortedViewIterator.java index fcfefa12f..062c462da 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/GenericSortedViewIterator.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/GenericSortedViewIterator.java @@ -23,11 +23,17 @@ /** * Iterator over quantile sketches of generic type. - * @param The generic quantile type + * @param The generic item class type */ public class GenericSortedViewIterator extends SortedViewIterator { private final T[] quantiles; + /** + * Constructor + * @param quantiles the given array of quantiles + * @param cumWeights the array of cumulative weights, corresponding to the array of quantiles, + * starting with the value one and the end value must equal N, the total number of items input to the sketch. + */ public GenericSortedViewIterator(final T[] quantiles, final long[] cumWeights) { super(cumWeights); this.quantiles = quantiles; //SpotBugs EI_EXPOSE_REP2 suppressed by FindBugsExcludeFilter diff --git a/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java b/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java index 3394a096a..203e338d2 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java @@ -27,36 +27,71 @@ */ public class IncludeMinMax { + /** A simple structure to hold a pair of arrays */ public static class DoublesPair { + /** the array of quantiles */ public double[] quantiles; + /** the array of associated cumulative weights */ public long[] cumWeights; + /** + * Constructor. + * @param quantiles the array of quantiles + * @param cumWeights the array of associated cumulative weights + */ public DoublesPair(final double[] quantiles, final long[] cumWeights) { this.quantiles = quantiles; this.cumWeights = cumWeights; } } + /** A simple structure to hold a pair of arrays */ public static class FloatsPair { + /** The array of quantiles */ public float[] quantiles; + /** The array of associated cumulative weights */ public long[] cumWeights; + /** + * Constructor. + * @param quantiles the array of quantiles + * @param cumWeights the array of associated cumulative weights + */ public FloatsPair(final float[] quantiles, final long[] cumWeights) { this.quantiles = quantiles; this.cumWeights = cumWeights; } } + /** + * A simple structure to hold a pair of arrays + * @param the item class type + */ public static class ItemsPair { + /** The array of quantiles */ public T[] quantiles; + /** The array of associated cumulative weights */ public long[] cumWeights; + /** + * Constructor. + * @param quantiles the array of quantiles + * @param cumWeights the array of associated cumulative weights + */ public ItemsPair(final T[] quantiles, final long[] cumWeights) { this.quantiles = quantiles; this.cumWeights = cumWeights; } } + /** + * The logic to include the min and max of type double. + * @param quantilesIn The array of quantiles + * @param cumWeightsIn The array of associated cumulative weights + * @param maxItem the maximum item of the stream + * @param minItem the minimum item of the stream + * @return a DoublesPair + */ public static DoublesPair includeDoublesMinMax( final double[] quantilesIn, final long[] cumWeightsIn, @@ -96,6 +131,14 @@ public static DoublesPair includeDoublesMinMax( return new DoublesPair(adjQuantiles, adjCumWeights); } + /** + * The logic to include the min and max of type float. + * @param quantilesIn The array of quantiles + * @param cumWeightsIn The array of associated cumulative weights + * @param maxItem the maximum item of the stream + * @param minItem the minimum item of the stream + * @return a FloatsPair + */ public static FloatsPair includeFloatsMinMax( final float[] quantilesIn, final long[] cumWeightsIn, @@ -135,6 +178,16 @@ public static FloatsPair includeFloatsMinMax( return new FloatsPair(adjQuantiles, adjCumWeights); } + /** + * The logic to include the min and max of type T. + * @param quantilesIn The array of quantiles + * @param cumWeightsIn The array of associated cumulative weights + * @param maxItem the maximum item of the stream + * @param minItem the minimum item of the stream + * @param comparator a comparator for type T + * @param the item class type + * @return an ItemsPair + */ @SuppressWarnings("unchecked") public static ItemsPair includeItemsMinMax( final T[] quantilesIn, diff --git a/src/main/java/org/apache/datasketches/quantilescommon/ItemsSketchSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/ItemsSketchSortedView.java index 4af4acd58..515783825 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/ItemsSketchSortedView.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/ItemsSketchSortedView.java @@ -49,9 +49,10 @@ public class ItemsSketchSortedView implements GenericSortedView { private final int numRetItems; /** - * Construct Sorted View. - * @param quantiles sorted array of quantiles - * @param cumWeights sorted, monotonically increasing cumulative weights. + * Constructor. + * @param quantiles the given array of quantiles, which must be ordered. + * @param cumWeights the given array of cumulative weights, which must be ordered, start with the value one, and + * the last value must be equal to N, the total number of items updated to the sketch. * @param sk the underlying quantile sketch. */ public ItemsSketchSortedView( @@ -198,6 +199,12 @@ private int getQuantileIndex(final double normRank, final long[] localCumWeights return index; } + /** + * Gets an array of quantiles corresponding to the given array of ranks. + * @param ranks the given array of normalized ranks + * @param searchCrit The search criterion: either INCLUSIVE or EXCLUSIVE. + * @return an array of quantiles corresponding to the given array of ranks. + */ @SuppressWarnings("unchecked") public T[] getQuantiles(final double[] ranks, final QuantileSearchCriteria searchCrit) { if (isEmpty()) { throw new IllegalArgumentException(QuantilesAPI.EMPTY_MSG); } diff --git a/src/main/java/org/apache/datasketches/quantilescommon/PartitioningFeature.java b/src/main/java/org/apache/datasketches/quantilescommon/PartitioningFeature.java index 380f57f08..5672c2a02 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/PartitioningFeature.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/PartitioningFeature.java @@ -23,6 +23,7 @@ /** * This enables the special functions for performing efficient partitioning of massive data. + * @param the item class type */ public interface PartitioningFeature { diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java index 0d02d2ec4..b70843bb4 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java @@ -202,6 +202,7 @@ * @author Kevin Lang * @author Alexander Saydakov */ +@SuppressWarnings("javadoc") public interface QuantilesAPI { static String EMPTY_MSG = "The sketch must not be empty for this operation. "; diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesUtil.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesUtil.java index a35aa27cd..75798c20f 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesUtil.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesUtil.java @@ -209,8 +209,16 @@ public static double[] evenlyLogSpaced(final double value1, final double value2, return arr; } + /** used in search to improve rounding over a wide dynamic range */ public static final double tailRoundingFactor = 1e7; + /** + * Computes the closest Natural Rank from a given Normalized Rank + * @param normalizedRank the given normalized rank + * @param totalN the total N + * @param searchCrit the search criterion. + * @return the closest Natural Rank from a given Normalized Rank + */ public static double getNaturalRank( final double normalizedRank, final long totalN, diff --git a/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java b/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java index f43c68cc8..b3aac54b4 100644 --- a/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java +++ b/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java @@ -46,7 +46,7 @@ * *

The sample may be smaller than k and the resulting size of the sample potentially includes * a probabilistic component, meaning the resulting sample size is not always constant. - * + * @param the item class type * @author Jon Malkin */ public final class EbppsItemsSketch { diff --git a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java index 2327cb135..1e3408511 100644 --- a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java +++ b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java @@ -43,6 +43,7 @@ */ public final class TDigestDouble { + /** the default value of K if one is not specified */ public static final short DEFAULT_K = 200; private boolean reverseMerge_; diff --git a/src/main/java/org/apache/datasketches/theta/BitPacking.java b/src/main/java/org/apache/datasketches/theta/BitPacking.java index 47e14d4ee..ca70dafa2 100644 --- a/src/main/java/org/apache/datasketches/theta/BitPacking.java +++ b/src/main/java/org/apache/datasketches/theta/BitPacking.java @@ -21,10 +21,20 @@ import org.apache.datasketches.common.SketchesArgumentException; +/** + * Used as part of Theta compression. + */ public class BitPacking { - public static void packBits(final long value, int bits, final byte[] buffer, int bufOffset, - final int bitOffset) { + /** + * The bit packing operation + * @param value the value to pack + * @param bits number of bits to pack + * @param buffer the output byte array buffer + * @param bufOffset the byte offset in the buffer + * @param bitOffset the bit offset + */ + public static void packBits(final long value, int bits, final byte[] buffer, int bufOffset, final int bitOffset) { if (bitOffset > 0) { final int chunkBits = 8 - bitOffset; final int mask = (1 << chunkBits) - 1; @@ -44,7 +54,16 @@ public static void packBits(final long value, int bits, final byte[] buffer, int } } - public static void unpackBits(final long[] value, final int index, int bits, final byte[] buffer, + /** + * The unpacking operation + * @param value the output array + * @param index index of the value array + * @param bits the number of bits to unpack + * @param buffer the input packed buffer + * @param bufOffset the buffer offset + * @param bitOffset the bit offset + */ + public static void unpackBits(final long[] value, final int index, int bits, final byte[] buffer, int bufOffset,final int bitOffset) { final int availBits = 8 - bitOffset; final int chunkBits = availBits <= bits ? availBits : bits; diff --git a/src/main/java/org/apache/datasketches/theta/CompactSketch.java b/src/main/java/org/apache/datasketches/theta/CompactSketch.java index 29d02b5cc..1426368f1 100644 --- a/src/main/java/org/apache/datasketches/theta/CompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/CompactSketch.java @@ -249,6 +249,10 @@ public boolean isCompact() { return true; } + /** + * gets the sketch as a compressed byte array + * @return the sketch as a compressed byte array + */ public byte[] toByteArrayCompressed() { if (!isOrdered() || getRetainedEntries() == 0 || (getRetainedEntries() == 1 && !isEstimationMode())) { return toByteArray(); diff --git a/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java b/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java index 0a6bd8bf7..20dd6ee7d 100644 --- a/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java +++ b/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java @@ -28,9 +28,11 @@ * Simplifies and speeds up set operations by resolving specific corner cases. * @author Lee Rhodes */ +@SuppressWarnings("javadoc") public class SetOperationCornerCases { private static final long MAX = Long.MAX_VALUE; + /** Intersection actions */ public enum IntersectAction { DEGEN_MIN_0_F("D", "Degenerate{MinTheta, 0, F}"), EMPTY_1_0_T("E", "Empty{1.0, 0, T}"), @@ -53,6 +55,7 @@ public String getActionDescription() { } } + /** A not B actions */ public enum AnotbAction { SKETCH_A("A", "Sketch A Exactly"), TRIM_A("TA", "Trim Sketch A by MinTheta"), diff --git a/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java b/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java index e4e603665..44d1d9cc0 100644 --- a/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java +++ b/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java @@ -31,6 +31,7 @@ public final class SerializerDeserializer { /** * Defines the sketch classes that this SerializerDeserializer can handle. */ + @SuppressWarnings("javadoc") public static enum SketchType { QuickSelectSketch, CompactSketch, ArrayOfDoublesQuickSelectSketch, ArrayOfDoublesCompactSketch, ArrayOfDoublesUnion } diff --git a/src/main/java/org/apache/datasketches/tuple/Util.java b/src/main/java/org/apache/datasketches/tuple/Util.java index c4f8ff55f..88fef2602 100644 --- a/src/main/java/org/apache/datasketches/tuple/Util.java +++ b/src/main/java/org/apache/datasketches/tuple/Util.java @@ -153,6 +153,13 @@ public static S[] copySummaryArray(final S[] summaryArr) { return tmpSummaryArr; } + /** + * Creates a new Summary Array with the specified length + * @param summaryArr example array, only used to obtain the component type. It has no data. + * @param length the desired length of the returned array. + * @param the summary class type + * @return a new Summary Array with the specified length + */ @SuppressWarnings("unchecked") public static S[] newSummaryArray(final S[] summaryArr, final int length) { final Class summaryType = (Class) summaryArr.getClass().getComponentType(); From 6c469cb197c960673c51a00694051d87a4474da8 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Tue, 30 Apr 2024 12:05:39 -0700 Subject: [PATCH 03/88] Fix pom header --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index da2d60b2f..35107f24c 100644 --- a/pom.xml +++ b/pom.xml @@ -18,9 +18,9 @@ specific language governing permissions and limitations under the License. --> - 4.0.0 From 271ad4029f13f079c7243a20592403557df33c3b Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 10 May 2024 10:36:46 -0700 Subject: [PATCH 04/88] Added new method to compute absolute maximum number of storage bytes required for a CompactSketch given the configured number of nominal entries (power of 2). --- .../java/org/apache/datasketches/theta/Sketch.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/org/apache/datasketches/theta/Sketch.java b/src/main/java/org/apache/datasketches/theta/Sketch.java index 888116512..960878fcb 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketch.java +++ b/src/main/java/org/apache/datasketches/theta/Sketch.java @@ -304,6 +304,18 @@ public static int getMaxCompactSketchBytes(final int numberOfEntries) { return (numberOfEntries << 3) + 24; } + /** + * Returns the maximum number of storage bytes required for a CompactSketch given the configured + * number of nominal entries (power of 2). + * @param nomEntries Nominal Entries + * @return the maximum number of storage bytes required for a CompactSketch with the given + * nomEntries. + */ + public static int getCompactSketchMaxBytes(final int nomEntries) { + final int nomEnt = ceilingPowerOf2(nomEntries); + return ((nomEnt << 4) * 15) / 16 + (Family.QUICKSELECT.getMaxPreLongs() << 3); + } + /** * Returns the maximum number of storage bytes required for an UpdateSketch with the given * number of nominal entries (power of 2). From 980c132d3aa0caf36cdda0c6e750ca466b2b2d8b Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 10 May 2024 12:07:01 -0700 Subject: [PATCH 05/88] Add test for new method --- .../org/apache/datasketches/theta/Sketch.java | 2 ++ .../apache/datasketches/theta/Sketches.java | 27 +++++++++++++++---- .../datasketches/theta/SketchesTest.java | 6 +++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/apache/datasketches/theta/Sketch.java b/src/main/java/org/apache/datasketches/theta/Sketch.java index 960878fcb..b949169c0 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketch.java +++ b/src/main/java/org/apache/datasketches/theta/Sketch.java @@ -297,7 +297,9 @@ public double getLowerBound(final int numStdDev) { * @param numberOfEntries the actual number of entries stored with the CompactSketch. * @return the maximum number of storage bytes required for a CompactSketch with the given number * of entries. + * @deprecated as a public method. Use {@link #getCompactSketchMaxBytes(int) instead} */ + @Deprecated public static int getMaxCompactSketchBytes(final int numberOfEntries) { if (numberOfEntries == 0) { return 8; } if (numberOfEntries == 1) { return 16; } diff --git a/src/main/java/org/apache/datasketches/theta/Sketches.java b/src/main/java/org/apache/datasketches/theta/Sketches.java index a5862e4a4..4b1461876 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketches.java +++ b/src/main/java/org/apache/datasketches/theta/Sketches.java @@ -79,15 +79,32 @@ public static int getMaxAnotBResultBytes(final int maxNomEntries) { } /** - * Ref: {@link Sketch#getMaxCompactSketchBytes(int)} - * @param numberOfEntries Ref: {@link Sketch#getMaxCompactSketchBytes(int)}, - * {@code numberOfEntries} - * @return Ref: {@link Sketch#getMaxCompactSketchBytes(int)} - */ + * Returns the maximum number of storage bytes required for a CompactSketch with the given + * number of actual entries. Note that this assumes the worse case of the sketch in + * estimation mode, which requires storing theta and count. + * @param numberOfEntries the actual number of entries stored with the CompactSketch. + * @return the maximum number of storage bytes required for a CompactSketch with the given number + * of entries. + * @see Sketch#getMaxCompactSketchBytes(int) + * @deprecated as a public method. Use {@link #getCompactSketchMaxBytes(int) instead} + */ + @Deprecated public static int getMaxCompactSketchBytes(final int numberOfEntries) { return Sketch.getMaxCompactSketchBytes(numberOfEntries); } + /** + * Returns the maximum number of storage bytes required for a CompactSketch given the configured + * number of nominal entries (power of 2). + * @param nomEntries Nominal Entries + * @return the maximum number of storage bytes required for a CompactSketch with the given + * nomEntries. + * @see Sketch#getCompactSketchMaxBytes(int) + */ + public static int getCompactSketchMaxBytes(final int nomEntries) { + return Sketch.getCompactSketchMaxBytes(nomEntries); + } + /** * Ref: {@link SetOperation#getMaxIntersectionBytes(int)} * @param nomEntries Ref: {@link SetOperation#getMaxIntersectionBytes(int)}, {@code nomEntries} diff --git a/src/test/java/org/apache/datasketches/theta/SketchesTest.java b/src/test/java/org/apache/datasketches/theta/SketchesTest.java index 6b887448e..cd51b50ed 100644 --- a/src/test/java/org/apache/datasketches/theta/SketchesTest.java +++ b/src/test/java/org/apache/datasketches/theta/SketchesTest.java @@ -20,6 +20,7 @@ package org.apache.datasketches.theta; import static org.apache.datasketches.theta.BackwardConversions.convertSerVer3toSerVer1; +import static org.apache.datasketches.theta.Sketches.getCompactSketchMaxBytes; import static org.apache.datasketches.theta.Sketches.getMaxCompactSketchBytes; import static org.apache.datasketches.theta.Sketches.getMaxIntersectionBytes; import static org.apache.datasketches.theta.Sketches.getMaxUnionBytes; @@ -35,6 +36,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; @@ -141,6 +143,10 @@ public void checkUtilMethods() { final int maxCompSkBytes = getMaxCompactSketchBytes(k+1); assertEquals(24+(k+1)*8, maxCompSkBytes); + final int compSkMaxBytes = getCompactSketchMaxBytes(k); { + assertEquals(compSkMaxBytes, ((k << 4) * 15) / 16 + (Family.QUICKSELECT.getMaxPreLongs() << 3)); + } + final int maxSkBytes = getMaxUpdateSketchBytes(k); assertEquals(24+2*k*8, maxSkBytes); } From 5c238e44f0919fef7a3cddb0e76568baacf8cf75 Mon Sep 17 00:00:00 2001 From: AlexanderSaydakov Date: Mon, 13 May 2024 19:20:47 -0700 Subject: [PATCH 06/88] removed intermediate hash map --- .../org/apache/datasketches/tuple/Filter.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/apache/datasketches/tuple/Filter.java b/src/main/java/org/apache/datasketches/tuple/Filter.java index 28b38295c..2ed156b84 100644 --- a/src/main/java/org/apache/datasketches/tuple/Filter.java +++ b/src/main/java/org/apache/datasketches/tuple/Filter.java @@ -19,10 +19,10 @@ package org.apache.datasketches.tuple; +import java.lang.reflect.Array; +import java.util.Arrays; import java.util.function.Predicate; -import org.apache.datasketches.common.ResizeFactor; - /** * Class for filtering entries from a {@link Sketch} given a {@link Summary} * @@ -52,23 +52,25 @@ public CompactSketch filter(final Sketch sketchIn) { if (sketchIn == null) { return new CompactSketch<>(null, null, Long.MAX_VALUE, true); } - - final QuickSelectSketch sketch = - new QuickSelectSketch<>(sketchIn.getRetainedEntries(), ResizeFactor.X1.lg(), null); + final long[] hashes = new long[sketchIn.getRetainedEntries()]; + T[] summaries = null; // lazy init to get class from the first entry + int i = 0; final TupleSketchIterator it = sketchIn.iterator(); while (it.next()) { final T summary = it.getSummary(); if (predicate.test(summary)) { - sketch.insert(it.getHash(), (T)summary.copy()); + hashes[i] = it.getHash(); + if (summaries == null) { + summaries = (T[]) Array.newInstance(summary.getClass(), sketchIn.getRetainedEntries()); + } + summaries[i++] = (T) summary.copy(); } } - - sketch.setThetaLong(sketchIn.getThetaLong()); - if (!sketchIn.isEmpty()) { - sketch.setEmpty(false); + final boolean isEmpty = i == 0 && !sketchIn.isEstimationMode(); + if (i == 0) { + return new CompactSketch<>(null, null, sketchIn.getThetaLong(), isEmpty); } - - return sketch.compact(); + return new CompactSketch<>(Arrays.copyOf(hashes, i), Arrays.copyOf(summaries, i), sketchIn.getThetaLong(), isEmpty); } } From 5a3144e3df677aa0a445770ca46f9cf57635c735 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Tue, 14 May 2024 17:12:24 -0700 Subject: [PATCH 07/88] Update SketchesCheckstyle Improvements to getCompactSketchMaxBytes(...) Renaming of two special functions one in HeapQuickSelectSketch and the other in DirectQuickSelectSketchR. Also changed their access privilege. --- .../theta/DirectQuickSelectSketch.java | 10 +++---- .../theta/DirectQuickSelectSketchR.java | 6 ++-- .../theta/HeapQuickSelectSketch.java | 10 +++---- .../org/apache/datasketches/theta/Sketch.java | 10 +++---- tools/SketchesCheckstyle.xml | 30 ++++++++++++------- 5 files changed, 38 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java index 8397c130b..a1ac53c6d 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java @@ -169,7 +169,7 @@ private DirectQuickSelectSketch( //clear hash table area dstMem.clear(preambleLongs << 3, 8 << lgArrLongs); - hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); + hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); memReqSvr_ = memReqSvr; } @@ -210,7 +210,7 @@ static DirectQuickSelectSketch writableWrap(final WritableMemory srcMem, final l final DirectQuickSelectSketch dqss = new DirectQuickSelectSketch(seed, srcMem); - dqss.hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); + dqss.hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); return dqss; } @@ -228,7 +228,7 @@ static DirectQuickSelectSketch fastWritableWrap(final WritableMemory srcMem, fin final DirectQuickSelectSketch dqss = new DirectQuickSelectSketch(seed, srcMem); - dqss.hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); + dqss.hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); return dqss; } @@ -310,7 +310,7 @@ UpdateReturnState hashUpdate(final long hash) { if (actLgRF > 0) { //Expand in current Memory //lgArrLongs will change; thetaLong, curCount will not resize(wmem_, preambleLongs, lgArrLongs, tgtLgArrLongs); - hashTableThreshold_ = setHashTableThreshold(lgNomLongs, tgtLgArrLongs); + hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, tgtLgArrLongs); return InsertedCountIncrementedResized; } //end of Expand in current memory, exit. @@ -330,7 +330,7 @@ UpdateReturnState hashUpdate(final long hash) { memReqSvr_.requestClose(wmem_, newDstMem); wmem_ = newDstMem; - hashTableThreshold_ = setHashTableThreshold(lgNomLongs, tgtLgArrLongs); + hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, tgtLgArrLongs); return InsertedCountIncrementedResized; } //end of Request more memory to resize } //end of resize diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java index c593f52e3..566d1b2e2 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java +++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java @@ -86,7 +86,7 @@ static DirectQuickSelectSketchR readOnlyWrap(final Memory srcMem, final long see final DirectQuickSelectSketchR dqssr = new DirectQuickSelectSketchR(seed, (WritableMemory) srcMem); - dqssr.hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); + dqssr.hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); return dqssr; } @@ -104,7 +104,7 @@ static DirectQuickSelectSketchR fastReadOnlyWrap(final Memory srcMem, final long final DirectQuickSelectSketchR dqss = new DirectQuickSelectSketchR(seed, (WritableMemory) srcMem); - dqss.hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); + dqss.hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); return dqss; } @@ -276,7 +276,7 @@ UpdateReturnState hashUpdate(final long hash) { * @return the hash table threshold */ @SuppressFBWarnings(value = "DB_DUPLICATE_BRANCHES", justification = "False Positive, see the code comments") - static final int setHashTableThreshold(final int lgNomLongs, final int lgArrLongs) { + protected static final int getOffHeapHashTableThreshold(final int lgNomLongs, final int lgArrLongs) { //SpotBugs may complain (DB_DUPLICATE_BRANCHES) if DQS_RESIZE_THRESHOLD == REBUILD_THRESHOLD, //but this allows us to tune these constants for different sketches. final double fraction = (lgArrLongs <= lgNomLongs) ? DQS_RESIZE_THRESHOLD : ThetaUtil.REBUILD_THRESHOLD; diff --git a/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java index 37b615456..627013f4f 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java @@ -92,7 +92,7 @@ private HeapQuickSelectSketch(final int lgNomLongs, final long seed, final float } lgArrLongs_ = ThetaUtil.startingSubMultiple(lgNomLongs + 1, rf.lg(), ThetaUtil.MIN_LG_ARR_LONGS); - hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs_); + hashTableThreshold_ = getHashTableThreshold(lgNomLongs, lgArrLongs_); curCount_ = 0; thetaLong_ = (long)(p * LONG_MAX_VALUE_AS_DOUBLE); empty_ = true; //other flags: bigEndian = readOnly = compact = ordered = false; @@ -128,7 +128,7 @@ static HeapQuickSelectSketch heapifyInstance(final Memory srcMem, final long see final HeapQuickSelectSketch hqss = new HeapQuickSelectSketch(lgNomLongs, seed, p, memRF, preambleLongs, family); hqss.lgArrLongs_ = lgArrLongs; - hqss.hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); + hqss.hashTableThreshold_ = getHashTableThreshold(lgNomLongs, lgArrLongs); hqss.curCount_ = extractCurCount(srcMem); hqss.thetaLong_ = extractThetaLong(srcMem); hqss.empty_ = PreambleUtil.isEmptyFlag(srcMem); @@ -197,7 +197,7 @@ public void reset() { cache_ = new long[1 << lgArrLongsSM]; lgArrLongs_ = lgArrLongsSM; } - hashTableThreshold_ = setHashTableThreshold(lgNomLongs_, lgArrLongs_); + hashTableThreshold_ = getHashTableThreshold(lgNomLongs_, lgArrLongs_); empty_ = true; curCount_ = 0; thetaLong_ = (long)(getP() * LONG_MAX_VALUE_AS_DOUBLE); @@ -293,7 +293,7 @@ private final void resizeCache() { curCount_ = newCount; cache_ = tgtArr; - hashTableThreshold_ = setHashTableThreshold(lgNomLongs_, lgArrLongs_); + hashTableThreshold_ = getHashTableThreshold(lgNomLongs_, lgArrLongs_); } //array stays the same size. Changes theta and thus count @@ -318,7 +318,7 @@ private final void quickSelectAndRebuild() { * @param lgArrLongs See lgArrLongs. * @return the hash table threshold */ - static final int setHashTableThreshold(final int lgNomLongs, final int lgArrLongs) { + private static final int getHashTableThreshold(final int lgNomLongs, final int lgArrLongs) { final double fraction = (lgArrLongs <= lgNomLongs) ? ThetaUtil.RESIZE_THRESHOLD : ThetaUtil.REBUILD_THRESHOLD; return (int) Math.floor(fraction * (1 << lgArrLongs)); } diff --git a/src/main/java/org/apache/datasketches/theta/Sketch.java b/src/main/java/org/apache/datasketches/theta/Sketch.java index b949169c0..cc1fd4d23 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketch.java +++ b/src/main/java/org/apache/datasketches/theta/Sketch.java @@ -308,14 +308,14 @@ public static int getMaxCompactSketchBytes(final int numberOfEntries) { /** * Returns the maximum number of storage bytes required for a CompactSketch given the configured - * number of nominal entries (power of 2). - * @param nomEntries Nominal Entries + * log_base2 of the number of nominal entries, which is a power of 2. + * @param lgNomEntries Nominal Entries * @return the maximum number of storage bytes required for a CompactSketch with the given * nomEntries. */ - public static int getCompactSketchMaxBytes(final int nomEntries) { - final int nomEnt = ceilingPowerOf2(nomEntries); - return ((nomEnt << 4) * 15) / 16 + (Family.QUICKSELECT.getMaxPreLongs() << 3); + public static int getCompactSketchMaxBytes(final int lgNomEntries) { + return (int)((2 << lgNomEntries) * ThetaUtil.REBUILD_THRESHOLD) + + Family.QUICKSELECT.getMaxPreLongs() * Long.BYTES; } /** diff --git a/tools/SketchesCheckstyle.xml b/tools/SketchesCheckstyle.xml index 8a45ea574..873a878a0 100644 --- a/tools/SketchesCheckstyle.xml +++ b/tools/SketchesCheckstyle.xml @@ -36,18 +36,17 @@ under the License. + + - + - - - - - - - - + + + + @@ -77,7 +76,18 @@ under the License. - + + + + + + + From ca32da73300df647446123da195305463fe29cb8 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Tue, 14 May 2024 17:21:28 -0700 Subject: [PATCH 08/88] Oops, forgot to update the test. --- .../java/org/apache/datasketches/theta/SketchesTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/apache/datasketches/theta/SketchesTest.java b/src/test/java/org/apache/datasketches/theta/SketchesTest.java index cd51b50ed..84942fe21 100644 --- a/src/test/java/org/apache/datasketches/theta/SketchesTest.java +++ b/src/test/java/org/apache/datasketches/theta/SketchesTest.java @@ -132,7 +132,8 @@ public void checkSetOpMethods() { @Test public void checkUtilMethods() { - final int k = 1024; + final int lgK = 10; + final int k = 1 << lgK; final int maxUnionBytes = getMaxUnionBytes(k); assertEquals(2*k*8+32, maxUnionBytes); @@ -143,8 +144,8 @@ public void checkUtilMethods() { final int maxCompSkBytes = getMaxCompactSketchBytes(k+1); assertEquals(24+(k+1)*8, maxCompSkBytes); - final int compSkMaxBytes = getCompactSketchMaxBytes(k); { - assertEquals(compSkMaxBytes, ((k << 4) * 15) / 16 + (Family.QUICKSELECT.getMaxPreLongs() << 3)); + final int compSkMaxBytes = getCompactSketchMaxBytes(lgK); { + assertEquals(compSkMaxBytes, ((2 << lgK) * 15) / 16 + (Family.QUICKSELECT.getMaxPreLongs() << 3)); } final int maxSkBytes = getMaxUpdateSketchBytes(k); From f979b373b9cdbaae994390f5f43caf976ec180de Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 15 May 2024 15:27:43 -0700 Subject: [PATCH 09/88] modified HeapQuickSelectSketch::getHashTableThreshold(..) And DirectQuickSelectSketchR::getOffHeapHashTableThreshold(..) to eliminate the unnecessary Math.floor(..) function. --- .../org/apache/datasketches/theta/DirectQuickSelectSketchR.java | 2 +- .../org/apache/datasketches/theta/HeapQuickSelectSketch.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java index 566d1b2e2..a3ffebc14 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java +++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java @@ -280,7 +280,7 @@ protected static final int getOffHeapHashTableThreshold(final int lgNomLongs, fi //SpotBugs may complain (DB_DUPLICATE_BRANCHES) if DQS_RESIZE_THRESHOLD == REBUILD_THRESHOLD, //but this allows us to tune these constants for different sketches. final double fraction = (lgArrLongs <= lgNomLongs) ? DQS_RESIZE_THRESHOLD : ThetaUtil.REBUILD_THRESHOLD; - return (int) Math.floor(fraction * (1 << lgArrLongs)); + return (int) (fraction * (1 << lgArrLongs)); } } diff --git a/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java index 627013f4f..b9d4dc9e1 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java @@ -320,7 +320,7 @@ private final void quickSelectAndRebuild() { */ private static final int getHashTableThreshold(final int lgNomLongs, final int lgArrLongs) { final double fraction = (lgArrLongs <= lgNomLongs) ? ThetaUtil.RESIZE_THRESHOLD : ThetaUtil.REBUILD_THRESHOLD; - return (int) Math.floor(fraction * (1 << lgArrLongs)); + return (int) (fraction * (1 << lgArrLongs)); } } From 8cea2d885cd8d14e2e9a586eaec48bd3602d23ac Mon Sep 17 00:00:00 2001 From: AlexanderSaydakov Date: Thu, 23 May 2024 12:06:54 -0700 Subject: [PATCH 10/88] get ID from Family --- src/main/java/org/apache/datasketches/req/ReqSerDe.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/req/ReqSerDe.java b/src/main/java/org/apache/datasketches/req/ReqSerDe.java index 079c9f282..52b1371a9 100644 --- a/src/main/java/org/apache/datasketches/req/ReqSerDe.java +++ b/src/main/java/org/apache/datasketches/req/ReqSerDe.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.datasketches.common.Family; import org.apache.datasketches.memory.Buffer; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableBuffer; @@ -126,7 +127,7 @@ class ReqSerDe { enum SerDeFormat { EMPTY, RAWITEMS, EXACT, ESTIMATION } private static final byte SER_VER = 1; - private static final byte FAMILY_ID = 17; + private static final byte FAMILY_ID = (byte) Family.REQ.getID(); static ReqSketch heapify(final Memory mem) { final Buffer buff = mem.asBuffer(); From e28cc2369e827d2c05cce6593a0fae36ea52cd69 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 29 May 2024 08:59:39 -0700 Subject: [PATCH 11/88] correct spelling --- .../apache/datasketches/filters/bloomfilter/BloomFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java index 3ea73b9bd..eafe88340 100644 --- a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java +++ b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java @@ -100,7 +100,7 @@ public final class BloomFilter { throw new SketchesArgumentException("Provided WritableMemory capacity insufficient to initialize BloomFilter"); } - // we don't resize so initialize wizth non-empty preLongs value + // we don't resize so initialize with non-empty preLongs value // and no empty flag final WritableBuffer wbuf = wmem.asWritableBuffer(); wbuf.putByte((byte) Family.BLOOMFILTER.getMaxPreLongs()); From 5c7f5abf9f58bebb9af0732bcec09dd7688d3dbf Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 31 May 2024 17:53:39 -0700 Subject: [PATCH 12/88] Changes in ds-java required to move to java 17 as ds-java 7.x and compatible with the new ds-memory branch integrateJava17_v3 -> ds-memory 3.X --- pom.xml | 10 +- .../apache/datasketches/cpc/CpcSketch.java | 2 +- .../datasketches/hll/DirectHllArray.java | 8 +- .../org/apache/datasketches/theta/Sketch.java | 6 +- .../apache/datasketches/theta/Sketches.java | 12 +- .../filters/bloomfilter/BloomFilterTest.java | 13 +- .../bloomfilter/DirectBitArrayRTest.java | 4 +- .../bloomfilter/DirectBitArrayTest.java | 12 +- ...ash3v2Test.java => MurmurHash3v3Test.java} | 75 ++++---- .../hll/DirectAuxHashMapTest.java | 14 +- .../hll/DirectCouponListTest.java | 11 +- .../datasketches/hll/PreambleUtilTest.java | 8 +- .../quantiles/DebugUnionTest.java | 12 +- .../DirectQuantilesMemoryRequestTest.java | 58 +++--- .../quantiles/DoublesSketchTest.java | 39 ++-- .../quantiles/PreambleUtilTest.java | 13 +- .../datasketches/theta/CompactSketchTest.java | 12 +- .../theta/DirectQuickSelectSketchTest.java | 166 +++++++++-------- .../theta/HeapifyWrapSerVer1and2Test.java | 171 +++++++++--------- .../datasketches/theta/SketchesTest.java | 2 +- .../datasketches/theta/UnionImplTest.java | 26 ++- .../datasketches/theta/UpdateSketchTest.java | 4 +- 22 files changed, 364 insertions(+), 314 deletions(-) rename src/test/java/org/apache/datasketches/hash/{MurmurHash3v2Test.java => MurmurHash3v3Test.java} (82%) diff --git a/pom.xml b/pom.xml index 35107f24c..e43a426ba 100644 --- a/pom.xml +++ b/pom.xml @@ -88,7 +88,7 @@ under the License. - 7.5.1 + 7.10.0 generate_java_files check_cpp_files @@ -96,7 +96,7 @@ under the License. 3.6.3 - 1.8 + 17 ${java.version} ${java.version} -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 @@ -134,12 +134,12 @@ under the License. - + @@ -193,7 +193,7 @@ under the License. - [1.8.0,9.0),[11.0,12.0) + [1.8.0,9.0),[11.0,12.0), [17,18) [${maven.version},) diff --git a/src/main/java/org/apache/datasketches/cpc/CpcSketch.java b/src/main/java/org/apache/datasketches/cpc/CpcSketch.java index a374b5745..4ed89dd19 100644 --- a/src/main/java/org/apache/datasketches/cpc/CpcSketch.java +++ b/src/main/java/org/apache/datasketches/cpc/CpcSketch.java @@ -300,7 +300,7 @@ public byte[] toByteArray() { final long cap = state.getRequiredSerializedBytes(); final WritableMemory wmem = WritableMemory.allocate((int) cap); state.exportToMemory(wmem); - return (byte[]) wmem.getArray(); + return wmem.getArray(); } /** diff --git a/src/main/java/org/apache/datasketches/hll/DirectHllArray.java b/src/main/java/org/apache/datasketches/hll/DirectHllArray.java index 5b3e1a4ff..07835da69 100644 --- a/src/main/java/org/apache/datasketches/hll/DirectHllArray.java +++ b/src/main/java/org/apache/datasketches/hll/DirectHllArray.java @@ -56,7 +56,7 @@ abstract class DirectHllArray extends AbstractHllArray { WritableMemory wmem; Memory mem; Object memObj; - long memAdd; + //long memAdd; final boolean compact; private static int checkMemCompactFlag(final WritableMemory wmem, final int lgConfigK) { @@ -70,7 +70,7 @@ private static int checkMemCompactFlag(final WritableMemory wmem, final int lgCo this.wmem = wmem; mem = wmem; memObj = wmem.getArray(); - memAdd = wmem.getCumulativeOffset(0L); + //memAdd = wmem.getCumulativeOffset(0L); compact = extractCompactFlag(mem); insertEmptyFlag(wmem, false); } @@ -81,7 +81,7 @@ private static int checkMemCompactFlag(final WritableMemory wmem, final int lgCo wmem = null; this.mem = mem; memObj = ((WritableMemory) mem).getArray(); - memAdd = mem.getCumulativeOffset(0L); + //memAdd = mem.getCumulativeOffset(0L); compact = extractCompactFlag(mem); } @@ -90,7 +90,7 @@ final void updateMemory(final WritableMemory newWmem) { wmem = newWmem; mem = newWmem; memObj = wmem.getArray(); - memAdd = wmem.getCumulativeOffset(0L); + //memAdd = wmem.getCumulativeOffset(0L); } @Override diff --git a/src/main/java/org/apache/datasketches/theta/Sketch.java b/src/main/java/org/apache/datasketches/theta/Sketch.java index cc1fd4d23..d13fe9b80 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketch.java +++ b/src/main/java/org/apache/datasketches/theta/Sketch.java @@ -311,11 +311,11 @@ public static int getMaxCompactSketchBytes(final int numberOfEntries) { * log_base2 of the number of nominal entries, which is a power of 2. * @param lgNomEntries Nominal Entries * @return the maximum number of storage bytes required for a CompactSketch with the given - * nomEntries. + * lgNomEntries. */ public static int getCompactSketchMaxBytes(final int lgNomEntries) { - return (int)((2 << lgNomEntries) * ThetaUtil.REBUILD_THRESHOLD) - + Family.QUICKSELECT.getMaxPreLongs() * Long.BYTES; + return (int)((2 << lgNomEntries) * ThetaUtil.REBUILD_THRESHOLD + + Family.QUICKSELECT.getMaxPreLongs()) * Long.BYTES; } /** diff --git a/src/main/java/org/apache/datasketches/theta/Sketches.java b/src/main/java/org/apache/datasketches/theta/Sketches.java index 4b1461876..077d9d5ae 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketches.java +++ b/src/main/java/org/apache/datasketches/theta/Sketches.java @@ -86,7 +86,7 @@ public static int getMaxAnotBResultBytes(final int maxNomEntries) { * @return the maximum number of storage bytes required for a CompactSketch with the given number * of entries. * @see Sketch#getMaxCompactSketchBytes(int) - * @deprecated as a public method. Use {@link #getCompactSketchMaxBytes(int) instead} + * @deprecated as a public method. Use {@link #getCompactSketchMaxBytes(int) getCompactSketchMaxBytes(int)} instead. */ @Deprecated public static int getMaxCompactSketchBytes(final int numberOfEntries) { @@ -95,14 +95,14 @@ public static int getMaxCompactSketchBytes(final int numberOfEntries) { /** * Returns the maximum number of storage bytes required for a CompactSketch given the configured - * number of nominal entries (power of 2). - * @param nomEntries Nominal Entries + * log_base2 of the number of nominal entries, which is a power of 2. + * @param lgNomEntries Nominal Entries * @return the maximum number of storage bytes required for a CompactSketch with the given - * nomEntries. + * lgNomEntries. * @see Sketch#getCompactSketchMaxBytes(int) */ - public static int getCompactSketchMaxBytes(final int nomEntries) { - return Sketch.getCompactSketchMaxBytes(nomEntries); + public static int getCompactSketchMaxBytes(final int lgNomEntries) { + return Sketch.getCompactSketchMaxBytes(lgNomEntries); } /** diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java index dedaf9db3..7a0fbb7ff 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java @@ -24,14 +24,19 @@ import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; +import java.nio.ByteOrder; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesReadOnlyException; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; +import jdk.incubator.foreign.ResourceScope; + public class BloomFilterTest { @Test @@ -51,8 +56,10 @@ public void createNewFilterTest() throws Exception { assertFalse(bf1.isDirect()); assertFalse(bf1.isReadOnly()); - try (WritableHandle wh = WritableMemory.allocateDirect(sizeBytes)) { - final WritableMemory wmem = wh.getWritable(); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(sizeBytes, + new DefaultMemoryRequestServer())).scope()) { + final BloomFilter bf2 = new BloomFilter(numBits, numHashes, seed, wmem); assertTrue(bf2.isEmpty()); assertTrue(bf2.hasMemory()); diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java index 521019e62..bbedd2fb7 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java @@ -142,8 +142,8 @@ public void bitAddressOutOfBoundsNonEmptyTest() { final Memory mem = bitArrayToMemory(hba); DirectBitArrayR dba = DirectBitArrayR.wrap(mem, hba.isEmpty()); - assertThrows(AssertionError.class, () -> dba.getBit(-10)); - assertThrows(AssertionError.class, () -> dba.getBit(2048)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.getBit(-10)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.getBit(2048)); } @Test diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java index a45bcbb82..1df6cc9d9 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java @@ -172,12 +172,12 @@ public void bitAddresOutOfBoundsNonEmptyTest() { dba.getAndSetBit(i); } - assertThrows(AssertionError.class, () -> dba.getBit(-10)); - assertThrows(AssertionError.class, () -> dba.getBit(2048)); - assertThrows(AssertionError.class, () -> dba.setBit(-20)); - assertThrows(AssertionError.class, () -> dba.setBit(4096)); - assertThrows(AssertionError.class, () -> dba.getAndSetBit(-30)); - assertThrows(AssertionError.class, () -> dba.getAndSetBit(8192)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.getBit(-10)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.getBit(2048)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.setBit(-20)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.setBit(4096)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.getAndSetBit(-30)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.getAndSetBit(8192)); } @Test diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3v2Test.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3v3Test.java similarity index 82% rename from src/test/java/org/apache/datasketches/hash/MurmurHash3v2Test.java rename to src/test/java/org/apache/datasketches/hash/MurmurHash3v3Test.java index 23f369e63..8699a091a 100644 --- a/src/test/java/org/apache/datasketches/hash/MurmurHash3v2Test.java +++ b/src/test/java/org/apache/datasketches/hash/MurmurHash3v3Test.java @@ -28,13 +28,13 @@ import org.testng.annotations.Test; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.MurmurHash3v2; +import org.apache.datasketches.memory.internal.MurmurHash3v3; import org.apache.datasketches.memory.WritableMemory; /** * @author Lee Rhodes */ -public class MurmurHash3v2Test { +public class MurmurHash3v3Test { private Random rand = new Random(); private static final int trials = 1 << 20; @@ -154,33 +154,33 @@ private static final long[] hashV1(byte[] key, long seed) { } private static final long[] hashV2(long[] key, long seed) { - return MurmurHash3v2.hash(key, seed); + return MurmurHash3v3.hash(key, seed); } private static final long[] hashV2(int[] key2, long seed) { - return MurmurHash3v2.hash(key2, seed); + return MurmurHash3v3.hash(key2, seed); } private static final long[] hashV2(char[] key, long seed) { - return MurmurHash3v2.hash(key, seed); + return MurmurHash3v3.hash(key, seed); } private static final long[] hashV2(byte[] key, long seed) { - return MurmurHash3v2.hash(key, seed); + return MurmurHash3v3.hash(key, seed); } //V2 single primitives private static final long[] hashV2(long key, long seed, long[] out) { - return MurmurHash3v2.hash(key, seed, out); + return MurmurHash3v3.hash(key, seed, out); } // private static final long[] hashV2(double key, long seed, long[] out) { -// return MurmurHash3v2.hash(key, seed, out); +// return MurmurHash3v3.hash(key, seed, out); // } // private static final long[] hashV2(String key, long seed, long[] out) { -// return MurmurHash3v2.hash(key, seed, out); +// return MurmurHash3v3.hash(key, seed, out); // } @@ -199,7 +199,7 @@ public void offsetChecks() { for (int offset = 0; offset < 16; offset++) { int arrLen = cap - offset; - hash1 = MurmurHash3v2.hash(wmem, offset, arrLen, seed, hash1); + hash1 = MurmurHash3v3.hash(wmem, offset, arrLen, seed, hash1); byte[] byteArr2 = new byte[arrLen]; wmem.getByteArray(offset, byteArr2, 0, arrLen); hash2 = MurmurHash3.hash(byteArr2, seed); @@ -222,8 +222,8 @@ public void byteArrChecks() { for (int i = 0; i < j; i++) { wmem.putByte(i, (byte) (-128 + i)); } long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v2.hash(in, seed); + hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v3.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -246,8 +246,8 @@ public void charArrChecks() { for (int i = 0; i < j; i++) { wmem.putInt(i, i); } long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v2.hash(in, seed); + hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v3.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -270,8 +270,8 @@ public void intArrChecks() { for (int i = 0; i < j; i++) { wmem.putInt(i, i); } long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v2.hash(in, seed); + hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v3.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -294,8 +294,8 @@ public void longArrChecks() { for (int i = 0; i < j; i++) { wmem.putLong(i, i); } long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v2.hash(in, seed); + hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v3.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -313,8 +313,8 @@ public void longCheck() { WritableMemory wmem = WritableMemory.writableWrap(in); long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v2.hash(in, seed); + hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v3.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -325,62 +325,57 @@ public void checkEmptiesNulls() { long seed = 123; long[] hashOut = new long[2]; try { - MurmurHash3v2.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut); //mem empty - fail(); - } catch (final IllegalArgumentException e) { } //OK - try { - Memory mem = null; - MurmurHash3v2.hash(mem, 0, 0, seed, hashOut); //mem null + MurmurHash3v3.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut); //mem empty fail(); } catch (final IllegalArgumentException e) { } //OK try { String s = ""; - MurmurHash3v2.hash(s, seed, hashOut); //string empty + MurmurHash3v3.hash(s, seed, hashOut); //string empty fail(); } catch (final IllegalArgumentException e) { } //OK try { String s = null; - MurmurHash3v2.hash(s, seed, hashOut); //string null + MurmurHash3v3.hash(s, seed, hashOut); //string null fail(); } catch (final IllegalArgumentException e) { } //OK try { byte[] barr = new byte[0]; - MurmurHash3v2.hash(barr, seed); //byte[] empty + MurmurHash3v3.hash(barr, seed); //byte[] empty fail(); } catch (final IllegalArgumentException e) { } //OK try { byte[] barr = null; - MurmurHash3v2.hash(barr, seed); //byte[] null + MurmurHash3v3.hash(barr, seed); //byte[] null fail(); } catch (final IllegalArgumentException e) { } //OK try { char[] carr = new char[0]; - MurmurHash3v2.hash(carr, seed); //char[] empty + MurmurHash3v3.hash(carr, seed); //char[] empty fail(); } catch (final IllegalArgumentException e) { } //OK try { char[] carr = null; - MurmurHash3v2.hash(carr, seed); //char[] null + MurmurHash3v3.hash(carr, seed); //char[] null fail(); } catch (final IllegalArgumentException e) { } //OK try { int[] iarr = new int[0]; - MurmurHash3v2.hash(iarr, seed); //int[] empty + MurmurHash3v3.hash(iarr, seed); //int[] empty fail(); } catch (final IllegalArgumentException e) { } //OK try { int[] iarr = null; - MurmurHash3v2.hash(iarr, seed); //int[] null + MurmurHash3v3.hash(iarr, seed); //int[] null fail(); } catch (final IllegalArgumentException e) { } //OK try { long[] larr = new long[0]; - MurmurHash3v2.hash(larr, seed); //long[] empty + MurmurHash3v3.hash(larr, seed); //long[] empty fail(); } catch (final IllegalArgumentException e) { } //OK try { long[] larr = null; - MurmurHash3v2.hash(larr, seed); //long[] null + MurmurHash3v3.hash(larr, seed); //long[] null fail(); } catch (final IllegalArgumentException e) { } //OK } @@ -390,9 +385,9 @@ public void checkStringLong() { long seed = 123; long[] hashOut = new long[2]; String s = "123"; - assertTrue(MurmurHash3v2.hash(s, seed, hashOut)[0] != 0); + assertTrue(MurmurHash3v3.hash(s, seed, hashOut)[0] != 0); long v = 123; - assertTrue(MurmurHash3v2.hash(v, seed, hashOut)[0] != 0); + assertTrue(MurmurHash3v3.hash(v, seed, hashOut)[0] != 0); } @Test @@ -420,8 +415,8 @@ private static long[] checkDouble(double dbl) { WritableMemory wmem = WritableMemory.writableWrap(dataArr); long[] hash1 = MurmurHash3.hash(dataArr, 0); - hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v2.hash(dbl, seed, hash2); + hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v3.hash(dbl, seed, hash2); assertEquals(hash1, hash2); assertEquals(hash1, hash3); diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java index 78b18c1cc..7b3cd5c67 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java @@ -31,12 +31,11 @@ import org.apache.datasketches.common.SketchesStateException; import org.apache.datasketches.memory.DefaultMemoryRequestServer; -import org.testng.annotations.Test; - import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; +import org.testng.annotations.Test; +import jdk.incubator.foreign.ResourceScope; /** * @author Lee Rhodes @@ -48,11 +47,12 @@ public void checkGrow() { int lgConfigK = 4; TgtHllType tgtHllType = TgtHllType.HLL_4; int n = 8; //put lgConfigK == 4 into HLL mode - int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType); + long bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType); HllSketch hllSketch; - try (WritableHandle handle = WritableMemory.allocateDirect(bytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - WritableMemory wmem = handle.getWritable(); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); for (int i = 0; i < n; i++) { hllSketch.update(i); diff --git a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java index 38cbc4977..c6c006f29 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java @@ -27,8 +27,11 @@ import org.testng.annotations.Test; +import jdk.incubator.foreign.ResourceScope; + +import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; /** @@ -69,9 +72,9 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool //println("DIRECT"); byte[] barr1; - WritableMemory wmem = null; - try (WritableHandle hand = WritableMemory.allocateDirect(bytes)) { - wmem = hand.getWritable(); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes, + new DefaultMemoryRequestServer())).scope()) { //byte[] byteArr = new byte[bytes]; //WritableMemory wmem = WritableMemory.wrap(byteArr); hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); diff --git a/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java b/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java index cd1e0cbcf..deb8c5be5 100644 --- a/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java +++ b/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java @@ -109,7 +109,7 @@ public void checkCorruptMemoryInput() { HllSketch sk = new HllSketch(12); byte[] memObj = sk.toCompactByteArray(); WritableMemory wmem = WritableMemory.writableWrap(memObj); - long memAdd = wmem.getCumulativeOffset(0); + //long memAdd = wmem.getCumulativeOffset(0); HllSketch bad; //checkFamily @@ -148,7 +148,7 @@ public void checkCorruptMemoryInput() { for (int i = 1; i <= 15; i++) { sk.update(i); } memObj = sk.toCompactByteArray(); wmem = WritableMemory.writableWrap(memObj); - memAdd = wmem.getCumulativeOffset(0); + //memAdd = wmem.getCumulativeOffset(0); //check wrong PreInts and SET try { @@ -162,7 +162,7 @@ public void checkCorruptMemoryInput() { for (int i = 15; i <= 1000; i++) { sk.update(i); } memObj = sk.toCompactByteArray(); wmem = WritableMemory.writableWrap(memObj); - memAdd = wmem.getCumulativeOffset(0); + //memAdd = wmem.getCumulativeOffset(0); //check wrong PreInts and HLL try { @@ -179,7 +179,7 @@ public void checkExtractFlags() { int bytes = HllSketch.getMaxUpdatableSerializationBytes(4, TgtHllType.HLL_4); WritableMemory wmem = WritableMemory.allocate(bytes); Object memObj = wmem.getArray(); - long memAdd = wmem.getCumulativeOffset(0L); + //long memAdd = wmem.getCumulativeOffset(0L); HllSketch sk = new HllSketch(4, TgtHllType.HLL_4, wmem); int flags = extractFlags(wmem); assertEquals(flags, EMPTY_FLAG_MASK); diff --git a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java index bbf89f338..d19d3b471 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java @@ -23,10 +23,15 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import java.nio.ByteOrder; import java.util.HashSet; import org.testng.annotations.Test; -import org.apache.datasketches.memory.WritableHandle; + +import jdk.incubator.foreign.ResourceScope; + +import org.apache.datasketches.memory.DefaultMemoryRequestServer; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.quantilescommon.QuantilesDoublesSketchIterator; @@ -61,8 +66,9 @@ public void test() { DoublesSketch.setRandom(1); //make deterministic for test DoublesUnion dUnion; DoublesSketch dSketch; - try ( WritableHandle wdh = WritableMemory.allocateDirect(10_000_000) ) { - WritableMemory wmem = wdh.getWritable(); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(10_000_000, + new DefaultMemoryRequestServer())).scope()) { dUnion = DoublesUnion.builder().setMaxK(8).build(wmem); for (int s = 0; s < numSketches; s++) { dUnion.union(sketchArr[s]); } dSketch = dUnion.getResult(); //result is on heap diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java index 5c2882c5e..84145239e 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java @@ -30,9 +30,11 @@ import org.testng.annotations.Test; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; +import jdk.incubator.foreign.ResourceScope; + /** * The concept for these tests is that the "MemoryManager" classes below are proxies for the * implementation that owns the native memory allocations, thus is responsible for @@ -47,10 +49,11 @@ public void checkLimitedMemoryScenarios() { //Requesting application final int initBytes = ((2 * k) + 4) << 3; //just the BB //########## Owning Implementation - // This part would actually be part of the Memory owning implemention so it is faked here - try (WritableHandle wdh = WritableMemory.allocateDirect(initBytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - final WritableMemory wmem = wdh.getWritable(); + // This part would actually be part of the Memory owning implementation so it is faked here + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + println("Initial mem size: " + wmem.getCapacity()); //########## Receiving Application @@ -84,13 +87,12 @@ public void checkGrowBaseBuf() { final int u = 32; // don't need the BB to fill here final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything - try (WritableHandle memHandler = WritableMemory.allocateDirect(initBytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - //final MemoryManager memMgr = new MemoryManager(); - //final WritableMemory mem1 = memMgr.request(initBytes); - final WritableMemory mem1 = memHandler.getWritable(); - println("Initial mem size: " + mem1.getCapacity()); - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + + println("Initial mem size: " + wmem.getCapacity()); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); for (int i = 1; i <= u; i++) { usk1.update(i); } @@ -108,13 +110,12 @@ public void checkGrowCombBuf() { final int u = (2 * k) - 1; //just to fill the BB final int initBytes = ((2 * k) + 4) << 3; //just room for BB - try (WritableHandle memHandler = WritableMemory.allocateDirect(initBytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - //final MemoryManager memMgr = new MemoryManager(); - //final WritableMemory mem1 = memMgr.request(initBytes); - final WritableMemory mem1 = memHandler.getWritable(); - println("Initial mem size: " + mem1.getCapacity()); - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + + println("Initial mem size: " + wmem.getCapacity()); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); for (int i = 1; i <= u; i++) { usk1.update(i); } @@ -138,21 +139,22 @@ public void checkGrowFromWrappedEmptySketch() { final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(); final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); - try (WritableHandle memHandle = WritableMemory.allocateDirect(initBytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - WritableMemory mem = memHandle.getWritable(); - origSketchMem.copyTo(0, mem, 0, initBytes); - UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(mem); - assertTrue(mem.isSameResource(usk2.getMemory())); - assertEquals(mem.getCapacity(), initBytes); - assertTrue(mem.isDirect()); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + + origSketchMem.copyTo(0, wmem, 0, initBytes); + UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem); + assertTrue(wmem.isSameResource(usk2.getMemory())); + assertEquals(wmem.getCapacity(), initBytes); + assertTrue(wmem.isDirect()); assertTrue(usk2.isEmpty()); //update the sketch forcing it to grow on-heap for (int i = 1; i <= 5; i++) { usk2.update(i); } assertEquals(usk2.getN(), 5); WritableMemory mem2 = usk2.getMemory(); - assertFalse(mem.isSameResource(mem2)); + assertFalse(wmem.isSameResource(mem2)); assertFalse(mem2.isDirect()); //should now be on-heap final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java index 5bc5b4eaa..fbac2af1e 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java @@ -28,13 +28,15 @@ import java.nio.ByteOrder; import org.apache.datasketches.memory.DefaultMemoryRequestServer; -import org.apache.datasketches.memory.WritableHandle; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.quantilescommon.DoublesSortedView; import org.apache.datasketches.quantilescommon.DoublesSortedViewIterator; import org.testng.Assert; import org.testng.annotations.Test; +import jdk.incubator.foreign.ResourceScope; + public class DoublesSketchTest { @Test @@ -140,15 +142,17 @@ public void checkEmptyExceptions() { @Test public void directSketchShouldMoveOntoHeapEventually() { - try (WritableHandle wdh = WritableMemory.allocateDirect(1000, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - WritableMemory mem = wdh.getWritable(); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); - Assert.assertTrue(sketch.isSameResource(mem)); + + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(1000, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); + Assert.assertTrue(sketch.isSameResource(wmem)); for (int i = 0; i < 1000; i++) { sketch.update(i); } - Assert.assertFalse(sketch.isSameResource(mem)); + Assert.assertFalse(sketch.isSameResource(wmem)); } catch (final Exception e) { throw new RuntimeException(e); } @@ -157,13 +161,14 @@ public void directSketchShouldMoveOntoHeapEventually() { @Test public void directSketchShouldMoveOntoHeapEventually2() { int i = 0; - try (WritableHandle wdh = - WritableMemory.allocateDirect(50, ByteOrder.LITTLE_ENDIAN, new DefaultMemoryRequestServer())) { - WritableMemory mem = wdh.getWritable(); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); - Assert.assertTrue(sketch.isSameResource(mem)); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(50, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); + Assert.assertTrue(sketch.isSameResource(wmem)); for (; i < 1000; i++) { - if (sketch.isSameResource(mem)) { + if (sketch.isSameResource(wmem)) { sketch.update(i); } else { //println("MOVED OUT at i = " + i); @@ -177,9 +182,11 @@ public void directSketchShouldMoveOntoHeapEventually2() { @Test public void checkEmptyDirect() { - try (WritableHandle wdh = WritableMemory.allocateDirect(1000)) { - WritableMemory mem = wdh.getWritable(); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(1000, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); sketch.toByteArray(); //exercises a specific path } catch (final Exception e) { throw new RuntimeException(e); diff --git a/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java b/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java index be2f328b2..adf916ef3 100644 --- a/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java @@ -37,19 +37,24 @@ import static org.apache.datasketches.quantiles.PreambleUtil.insertSerVer; import static org.testng.Assert.assertEquals; -import org.testng.annotations.Test; +import java.nio.ByteOrder; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; +import org.testng.annotations.Test; + +import jdk.incubator.foreign.ResourceScope; public class PreambleUtilTest { @Test public void checkInsertsAndExtracts() { final int bytes = 32; - try (WritableHandle offHeapMemHandler = WritableMemory.allocateDirect(bytes)) { - final WritableMemory offHeapMem = offHeapMemHandler.getWritable(); + WritableMemory offHeapMem; + try (ResourceScope scope = (offHeapMem = WritableMemory.allocateDirect(bytes, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + final WritableMemory onHeapMem = WritableMemory.writableWrap(new byte[bytes]); onHeapMem.clear(); diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java index 42b6069da..1ad55696c 100644 --- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java @@ -26,13 +26,18 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import java.nio.ByteOrder; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; +import jdk.incubator.foreign.ResourceScope; + /** * @author Lee Rhodes */ @@ -78,8 +83,9 @@ public void checkHeapifyWrap(int k, int u, boolean ordered) { //Prepare Memory for direct int bytes = usk.getCompactBytes(); //for Compact - try (WritableHandle wdh = WritableMemory.allocateDirect(bytes)) { - WritableMemory directMem = wdh.getWritable(); + WritableMemory directMem; + try (ResourceScope scope = (directMem = WritableMemory.allocateDirect(bytes, + new DefaultMemoryRequestServer())).scope()) { /**Via CompactSketch.compact**/ refSk = usk.compact(ordered, directMem); diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java index 5191c7c68..1f1bfa9ec 100644 --- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java @@ -38,19 +38,23 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.nio.ByteOrder; import java.util.Arrays; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesReadOnlyException; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.thetacommon.HashOperations; import org.apache.datasketches.thetacommon.ThetaUtil; import org.testng.annotations.Test; +import jdk.incubator.foreign.ResourceScope; + /** * @author Lee Rhodes */ @@ -59,10 +63,10 @@ public class DirectQuickSelectSketchTest { @Test//(expectedExceptions = SketchesArgumentException.class) public void checkBadSerVer() { int k = 512; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); + WritableMemory wmem; + try (ResourceScope scope = (wmem = makeNativeMemory(k)).scope()) { - UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(mem); + UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(wmem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks assertTrue(usk.isEmpty()); @@ -73,9 +77,9 @@ public void checkBadSerVer() { assertEquals(usk.getEstimate(), k, 0.0); assertEquals(sk1.getRetainedEntries(false), k); - mem.putByte(SER_VER_BYTE, (byte) 0); //corrupt the SerVer byte + wmem.putByte(SER_VER_BYTE, (byte) 0); //corrupt the SerVer byte - Sketch.wrap(mem); + Sketch.wrap(wmem); } catch (final Exception e) { if (e instanceof SketchesArgumentException) {} else { throw new RuntimeException(e); } @@ -85,9 +89,9 @@ public void checkBadSerVer() { @Test//(expectedExceptions = SketchesArgumentException.class) public void checkConstructorKtooSmall() { int k = 8; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); - UpdateSketch.builder().setNominalEntries(k).build(mem); + WritableMemory wmem; + try (ResourceScope scope = (wmem = makeNativeMemory(k)).scope()) { + UpdateSketch.builder().setNominalEntries(k).build(wmem); } catch (final Exception e) { if (e instanceof SketchesArgumentException) {} else { throw new RuntimeException(e); } @@ -97,9 +101,9 @@ public void checkConstructorKtooSmall() { @Test//(expectedExceptions = SketchesArgumentException.class) public void checkConstructorMemTooSmall() { int k = 16; - try (WritableHandle h = makeNativeMemory(k/2)) { - WritableMemory mem = h.getWritable(); - UpdateSketch.builder().setNominalEntries(k).build(mem); + WritableMemory wmem; + try (ResourceScope scope = (wmem = makeNativeMemory(k/2)).scope()) { + UpdateSketch.builder().setNominalEntries(k).build(wmem); } catch (final Exception e) { if (e instanceof SketchesArgumentException) {} else { throw new RuntimeException(e); } @@ -124,10 +128,10 @@ public void checkHeapifyMemoryEstimating() { int k = 512; int u = 2*k; //thus estimating - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); + WritableMemory wmem; + try (ResourceScope scope = (wmem = makeNativeMemory(k)).scope()) { - UpdateSketch sk1 = UpdateSketch.builder().setNominalEntries(k).build(mem); + UpdateSketch sk1 = UpdateSketch.builder().setNominalEntries(k).build(wmem); for (int i=0; i Date: Fri, 14 Jun 2024 17:12:19 -0700 Subject: [PATCH 13/88] Fixes to align with modified Memory --- .github/workflows/auto-jdk-matrix.yml | 41 ++--- .../hll/DirectAuxHashMapTest.java | 83 +++++----- .../DirectQuantilesMemoryRequestTest.java | 147 ++++++++---------- .../quantiles/DoublesSketchTest.java | 43 ++--- .../theta/DirectQuickSelectSketchTest.java | 64 ++++---- .../datasketches/theta/UnionImplTest.java | 25 ++- 6 files changed, 179 insertions(+), 224 deletions(-) diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml index 83da580ca..ce15bb57e 100644 --- a/.github/workflows/auto-jdk-matrix.yml +++ b/.github/workflows/auto-jdk-matrix.yml @@ -7,49 +7,51 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Xmx4g -Xms1g + MAVEN_OPTS: -Xmx1g -Xms1g jobs: build: name: Build, Test, Install runs-on: ubuntu-latest + strategy: fail-fast: false - matrix: - jdk: [ 8,11 ] + env: - JDK_VERSION: ${{ matrix.jdk }} + JDK_VERSION: 17 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false + - name: Print Current workflow + run: > + cat .github/workflows/auto-jdk-matrix.yml + + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-package: jdk + architecture: x64 + java-version: 17 + - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: build-${{ runner.os }}-maven- - - name: Install Matrix JDK - uses: actions/setup-java@v3 - with: - java-version: ${{ matrix.jdk }} - distribution: 'temurin' - java-package: jdk - architecture: x64 -# Architecture options: x86, x64, armv7, aarch64, ppc64le -# setup-java@v3 has a "with cache" option - - name: Echo Java Version run: > java -version - name: Test run: > - mvn clean test + mvn clean test -B -Dmaven.javadoc.skip=true -Dgpg.skip=true @@ -59,7 +61,10 @@ jobs: -DskipTests=true -Dgpg.skip=true +# Architecture options: x86, x64, armv7, aarch64, ppc64le +# setup-java@v4 has a "with cache" option # Lifecycle: validate, compile, test, package, verify, install, deploy -# -B batch mode +# -B batch mode, never stops for user input # -V show Version without stopping +# -X debug mode # -q quiet, only show errors diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java index 7b3cd5c67..98039616f 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java @@ -49,52 +49,47 @@ public void checkGrow() { int n = 8; //put lgConfigK == 4 into HLL mode long bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType); HllSketch hllSketch; - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + WritableMemory wmem = WritableMemory.allocateDirect(bytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); - for (int i = 0; i < n; i++) { - hllSketch.update(i); - } - hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values - hllSketch.couponUpdate(HllUtil.pair(8, 15)); - hllSketch.couponUpdate(HllUtil.pair(9, 15)); - //println(hllSketch.toString(true, true, true, true)); - DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl; - assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2); - assertTrue(hllSketch.isMemory()); - assertTrue(hllSketch.isOffHeap()); - assertTrue(hllSketch.isSameResource(wmem)); - - //Check heapify - byte[] byteArray = hllSketch.toCompactByteArray(); - HllSketch hllSketch2 = HllSketch.heapify(byteArray); - HllArray ha = (HllArray) hllSketch2.hllSketchImpl; - assertEquals(ha.getAuxHashMap().getLgAuxArrInts(), 2); - assertEquals(ha.getAuxHashMap().getAuxCount(), 3); - - //Check wrap - byteArray = hllSketch.toUpdatableByteArray(); - WritableMemory wmem2 = WritableMemory.writableWrap(byteArray); - hllSketch2 = HllSketch.writableWrap(wmem2); - //println(hllSketch2.toString(true, true, true, true)); - DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl; - assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2); - assertEquals(dha2.getAuxHashMap().getAuxCount(), 3); - - //Check grow to on-heap - hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow - //println(hllSketch.toString(true, true, true, true)); - dha = (DirectHllArray) hllSketch.hllSketchImpl; - assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3); - assertEquals(dha.getAuxHashMap().getAuxCount(), 4); - assertTrue(hllSketch.isMemory()); - assertFalse(hllSketch.isOffHeap()); - assertFalse(hllSketch.isSameResource(wmem)); - } catch (final Exception e) { - throw new RuntimeException(e); + hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); + for (int i = 0; i < n; i++) { + hllSketch.update(i); } + hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values + hllSketch.couponUpdate(HllUtil.pair(8, 15)); + hllSketch.couponUpdate(HllUtil.pair(9, 15)); + //println(hllSketch.toString(true, true, true, true)); + DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl; + assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2); + assertTrue(hllSketch.isMemory()); + assertTrue(hllSketch.isOffHeap()); + assertTrue(hllSketch.isSameResource(wmem)); + + //Check heapify + byte[] byteArray = hllSketch.toCompactByteArray(); + HllSketch hllSketch2 = HllSketch.heapify(byteArray); + HllArray ha = (HllArray) hllSketch2.hllSketchImpl; + assertEquals(ha.getAuxHashMap().getLgAuxArrInts(), 2); + assertEquals(ha.getAuxHashMap().getAuxCount(), 3); + + //Check wrap + byteArray = hllSketch.toUpdatableByteArray(); + WritableMemory wmem2 = WritableMemory.writableWrap(byteArray); + hllSketch2 = HllSketch.writableWrap(wmem2); + //println(hllSketch2.toString(true, true, true, true)); + DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl; + assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2); + assertEquals(dha2.getAuxHashMap().getAuxCount(), 3); + + //Check grow to on-heap + hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow + //println(hllSketch.toString(true, true, true, true)); + dha = (DirectHllArray) hllSketch.hllSketchImpl; + assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3); + assertEquals(dha.getAuxHashMap().getAuxCount(), 4); + assertTrue(hllSketch.isMemory()); + assertFalse(hllSketch.isOffHeap()); + assertFalse(hllSketch.isSameResource(wmem)); } @Test diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java index 84145239e..6552d0ae3 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java @@ -46,39 +46,34 @@ public class DirectQuantilesMemoryRequestTest { public void checkLimitedMemoryScenarios() { //Requesting application final int k = 128; final int u = 40 * k; - final int initBytes = ((2 * k) + 4) << 3; //just the BB + final int initBytes = ((2 * k) + 4) << 3; //just the BaseBuffer //########## Owning Implementation // This part would actually be part of the Memory owning implementation so it is faked here - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - - println("Initial mem size: " + wmem.getCapacity()); - - //########## Receiving Application - // The receiving application has been given wmem to use for a sketch, - // but alas, it is not ultimately large enough. - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); - assertTrue(usk1.isEmpty()); - - //Load the sketch - for (int i = 0; i < u; i++) { - // The sketch uses The MemoryRequest, acquired from wmem, to acquire more memory as - // needed, and requests via the MemoryRequest to free the old allocations. - usk1.update(i); - } - final double result = usk1.getQuantile(0.5); - println("Result: " + result); - assertEquals(result, u / 2.0, 0.05 * u); //Success - - //########## Owning Implementation - //The actual Memory has been re-allocated several times, - // so the above wmem reference is invalid. - println("\nFinal mem size: " + wmem.getCapacity()); - } catch (Exception e) { - throw new RuntimeException(e); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + + println("Initial mem size: " + wmem.getCapacity()); + + //########## Receiving Application + // The receiving application has been given wmem to use for a sketch, + // but alas, it is not ultimately large enough. + final UpdateDoublesSketch usk = DoublesSketch.builder().setK(k).build(wmem); + assertTrue(usk.isEmpty()); + + //Load the sketch + for (int i = 0; i < u; i++) { + // The sketch uses The MemoryRequest, acquired from wmem, to acquire more memory as + // needed, and requests via the MemoryRequest to free the old allocations. + usk.update(i); } + final double result = usk.getQuantile(0.5); + println("Result: " + result); + assertEquals(result, u / 2.0, 0.05 * u); //Success + + //########## Owning Implementation + //The actual Memory has been re-allocated several times, + // so the above wmem reference is invalid. + println("\nFinal mem size: " + wmem.getCapacity()); } @Test @@ -87,21 +82,16 @@ public void checkGrowBaseBuf() { final int u = 32; // don't need the BB to fill here final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - - println("Initial mem size: " + wmem.getCapacity()); - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); - for (int i = 1; i <= u; i++) { - usk1.update(i); - } - final int currentSpace = usk1.getCombinedBufferItemCapacity(); - println("curCombBufItemCap: " + currentSpace); - assertEquals(currentSpace, 2 * k); - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + + println("Initial mem size: " + wmem.getCapacity()); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); + for (int i = 1; i <= u; i++) { + usk1.update(i); } + final int currentSpace = usk1.getCombinedBufferItemCapacity(); + println("curCombBufItemCap: " + currentSpace); + assertEquals(currentSpace, 2 * k); } @Test @@ -110,25 +100,19 @@ public void checkGrowCombBuf() { final int u = (2 * k) - 1; //just to fill the BB final int initBytes = ((2 * k) + 4) << 3; //just room for BB - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - - println("Initial mem size: " + wmem.getCapacity()); - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); - for (int i = 1; i <= u; i++) { - usk1.update(i); - } - final int currentSpace = usk1.getCombinedBufferItemCapacity(); - println("curCombBufItemCap: " + currentSpace); - final double[] newCB = usk1.growCombinedBuffer(currentSpace, 3 * k); - final int newSpace = usk1.getCombinedBufferItemCapacity(); - println("newCombBurItemCap: " + newSpace); - assertEquals(newCB.length, 3 * k); - //memMgr.free(mem1); - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + + println("Initial mem size: " + wmem.getCapacity()); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); + for (int i = 1; i <= u; i++) { + usk1.update(i); } + final int currentSpace = usk1.getCombinedBufferItemCapacity(); + println("curCombBufItemCap: " + currentSpace); + final double[] newCB = usk1.growCombinedBuffer(currentSpace, 3 * k); + final int newSpace = usk1.getCombinedBufferItemCapacity(); + println("newCombBurItemCap: " + newSpace); + assertEquals(newCB.length, 3 * k); } @Test @@ -139,29 +123,24 @@ public void checkGrowFromWrappedEmptySketch() { final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(); final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - - origSketchMem.copyTo(0, wmem, 0, initBytes); - UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem); - assertTrue(wmem.isSameResource(usk2.getMemory())); - assertEquals(wmem.getCapacity(), initBytes); - assertTrue(wmem.isDirect()); - assertTrue(usk2.isEmpty()); - - //update the sketch forcing it to grow on-heap - for (int i = 1; i <= 5; i++) { usk2.update(i); } - assertEquals(usk2.getN(), 5); - WritableMemory mem2 = usk2.getMemory(); - assertFalse(wmem.isSameResource(mem2)); - assertFalse(mem2.isDirect()); //should now be on-heap - - final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); - assertEquals(mem2.getCapacity(), expectedSize); - } catch (final Exception e) { - throw new RuntimeException(e); - } + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + + origSketchMem.copyTo(0, wmem, 0, initBytes); + UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem); + assertTrue(wmem.isSameResource(usk2.getMemory())); + assertEquals(wmem.getCapacity(), initBytes); + assertTrue(wmem.isDirect()); + assertTrue(usk2.isEmpty()); + + //update the sketch forcing it to grow on-heap + for (int i = 1; i <= 5; i++) { usk2.update(i); } + assertEquals(usk2.getN(), 5); + WritableMemory mem2 = usk2.getMemory(); + assertFalse(wmem.isSameResource(mem2)); + assertFalse(mem2.isDirect()); //should now be on-heap + + final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); + assertEquals(mem2.getCapacity(), expectedSize); } @Test diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java index fbac2af1e..8f4243bfe 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java @@ -143,46 +143,35 @@ public void checkEmptyExceptions() { @Test public void directSketchShouldMoveOntoHeapEventually() { - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(1000, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - - UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); - Assert.assertTrue(sketch.isSameResource(wmem)); - for (int i = 0; i < 1000; i++) { - sketch.update(i); - } - Assert.assertFalse(sketch.isSameResource(wmem)); - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory wmem = WritableMemory.allocateDirect(1000, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); + Assert.assertTrue(sketch.isSameResource(wmem)); + for (int i = 0; i < 1000; i++) { + sketch.update(i); } + Assert.assertFalse(sketch.isSameResource(wmem)); } @Test public void directSketchShouldMoveOntoHeapEventually2() { int i = 0; - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(50, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + WritableMemory wmem = WritableMemory.allocateDirect(50, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); - Assert.assertTrue(sketch.isSameResource(wmem)); - for (; i < 1000; i++) { - if (sketch.isSameResource(wmem)) { - sketch.update(i); - } else { - //println("MOVED OUT at i = " + i); - break; - } + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); + Assert.assertTrue(sketch.isSameResource(wmem)); + for (; i < 1000; i++) { + if (sketch.isSameResource(wmem)) { + sketch.update(i); + } else { + //println("MOVED OUT at i = " + i); + break; } - } catch (final Exception e) { - throw new RuntimeException(e); } } @Test public void checkEmptyDirect() { - WritableMemory wmem; + WritableMemory wmem ; try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(1000, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java index 1f1bfa9ec..32b9bbb44 100644 --- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java @@ -876,16 +876,12 @@ public void checkMoveAndResize() { int u = 2 * k; int bytes = Sketches.getMaxUpdateSketchBytes(k); - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes / 2, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); - assertTrue(sketch.isSameResource(wmem)); - for (int i = 0; i < u; i++) { sketch.update(i); } - assertFalse(sketch.isSameResource(wmem)); - } catch (final Exception e) { - throw new RuntimeException(e); - } + WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + + UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); + assertTrue(sketch.isSameResource(wmem)); + for (int i = 0; i < u; i++) { sketch.update(i); } + assertFalse(sketch.isSameResource(wmem)); } @Test @@ -893,36 +889,30 @@ public void checkReadOnlyRebuildResize() { int k = 1 << 12; int u = 2 * k; int bytes = Sketches.getMaxUpdateSketchBytes(k); - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes / 2, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); - for (int i = 0; i < u; i++) { sketch.update(i); } - double est1 = sketch.getEstimate(); - byte[] ser = sketch.toByteArray(); - Memory mem = Memory.wrap(ser); - UpdateSketch roSketch = (UpdateSketch) Sketches.wrapSketch(mem); - double est2 = roSketch.getEstimate(); - assertEquals(est2, est1); - try { - roSketch.rebuild(); - fail(); - } catch (SketchesReadOnlyException e) { - //expected - } - try { - roSketch.reset(); - fail(); - } catch (SketchesReadOnlyException e) { - //expected - } - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + + UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); + for (int i = 0; i < u; i++) { sketch.update(i); } + double est1 = sketch.getEstimate(); + byte[] ser = sketch.toByteArray(); + Memory mem = Memory.wrap(ser); + UpdateSketch roSketch = (UpdateSketch) Sketches.wrapSketch(mem); + double est2 = roSketch.getEstimate(); + assertEquals(est2, est1); + try { + roSketch.rebuild(); + fail(); + } catch (SketchesReadOnlyException e) { + //expected + } + try { + roSketch.reset(); + fail(); + } catch (SketchesReadOnlyException e) { + //expected } - } - @Test public void printlnTest() { println("PRINTING: "+this.getClass().getName()); diff --git a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java index ee0fed44c..fe76240a7 100644 --- a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java +++ b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java @@ -197,24 +197,21 @@ public void checkMoveAndResize() { final int k = 1 << 12; final int u = 2 * k; final int bytes = Sketches.getMaxUpdateSketchBytes(k); - try (ResourceScope scope = ResourceScope.newConfinedScope()) { + ResourceScope scope = ResourceScope.newConfinedScope(); - final WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2, 1, scope, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); - assertTrue(sketch.isSameResource(wmem)); + final WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2, 1, scope, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); + assertTrue(sketch.isSameResource(wmem)); - final WritableMemory wmem2 = WritableMemory.allocateDirect(bytes / 2, 1, scope, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - final Union union = SetOperation.builder().buildUnion(wmem2); - assertTrue(union.isSameResource(wmem2)); + final WritableMemory wmem2 = WritableMemory.allocateDirect(bytes / 2, 1, scope, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + final Union union = SetOperation.builder().buildUnion(wmem2); + assertTrue(union.isSameResource(wmem2)); - for (int i = 0; i < u; i++) { union.update(i); } - assertFalse(union.isSameResource(wmem)); + for (int i = 0; i < u; i++) { union.update(i); } + assertFalse(union.isSameResource(wmem)); - final Union union2 = SetOperation.builder().buildUnion(); //on-heap union - assertFalse(union2.isSameResource(wmem2)); //obviously not - } catch (final Exception e) { - throw new RuntimeException(e); - } + final Union union2 = SetOperation.builder().buildUnion(); //on-heap union + assertFalse(union2.isSameResource(wmem2)); //obviously not } @Test From b984e6523343ad6b5143c6b17b461d91a57c45e5 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 20 Jun 2024 08:33:58 -0700 Subject: [PATCH 14/88] Changes required for ds-java to move to java17 and also depend on ds-Memory 4.0.0, which is built around FFM/Panama. --- pom.xml | 157 +++++------------- .../quantiles/DirectUpdateDoublesSketch.java | 3 +- .../filters/bloomfilter/BloomFilterTest.java | 3 +- .../hll/DirectCouponListTest.java | 3 +- .../quantiles/DebugUnionTest.java | 3 +- .../datasketches/theta/CompactSketchTest.java | 3 +- .../theta/DirectQuickSelectSketchTest.java | 5 +- .../theta/HeapifyWrapSerVer1and2Test.java | 2 +- .../datasketches/theta/UnionImplTest.java | 3 +- 9 files changed, 48 insertions(+), 134 deletions(-) diff --git a/pom.xml b/pom.xml index e43a426ba..949152680 100644 --- a/pom.xml +++ b/pom.xml @@ -18,19 +18,11 @@ specific language governing permissions and limitations under the License. --> - + 4.0.0 - - org.apache - apache - 32 - - org.apache.datasketches datasketches-java 6.1.0-SNAPSHOT @@ -66,8 +58,8 @@ under the License. - + + GitHub https://github.com/apache/${project.artifactId}/issues @@ -83,9 +75,7 @@ under the License. - 2.2.0 - 7.10.0 @@ -95,52 +85,51 @@ under the License. check_cpp_historical_files - 3.6.3 + 3.6.3 17 ${java.version} ${java.version} - -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 + -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 --add-modules=jdk.incubator.foreign UTF-8 ${charset.encoding} ${charset.encoding} ${charset.encoding} yyyy-MM-dd'T'HH-mm-ss'Z' - - 2.16.2 - - 3.7.1 - 3.13.0 - 3.1.1 - 3.4.1 - 3.2.3 - 3.4.0 - 3.6.3 - 3.0.1 - 3.2.0 - 3.3.1 - 3.2.5 + 3.7.1 + 3.13.0 + 3.1.1 + 3.4.1 + 3.2.3 + 3.4.0 + 3.7.0 + 3.0.1 + 3.2.0 + 3.3.1 + 3.2.5 + 3.1.2 - 0.16.1 + 0.16.1 0.8.12 4.3.0 + + 2.16.2 + - 1.0.0 + 1.0.0 4.9.10 - - + org.testng @@ -170,6 +159,11 @@ under the License. org.apache.maven.plugins maven-compiler-plugin ${maven-compiler-plugin.version} + + + --add-modules=jdk.incubator.foreign + + @@ -193,7 +187,7 @@ under the License. - [1.8.0,9.0),[11.0,12.0), [17,18) + [17,18) [${maven.version},) @@ -212,7 +206,6 @@ under the License. - org.apache.maven.plugins maven-jar-plugin ${maven-jar-plugin.version} @@ -235,12 +228,15 @@ under the License. - org.apache.maven.plugins maven-javadoc-plugin ${maven-javadoc-plugin.version} true + public + + --add-modules=jdk.incubator.foreign + @@ -253,14 +249,12 @@ under the License. - org.apache.maven.plugins maven-release-plugin ${maven-release-plugin.version} - org.apache.maven.plugins maven-source-plugin ${maven-source-plugin.version} @@ -283,11 +277,11 @@ under the License. - org.apache.maven.plugins maven-surefire-plugin ${maven-surefire-failsafe-plugins.version} + --add-modules=jdk.incubator.foreign false false true @@ -297,7 +291,6 @@ under the License. - org.apache.rat apache-rat-plugin ${apache-rat-plugin.version} @@ -423,44 +416,8 @@ under the License. - - - - only-eclipse - - - m2e.version - - - - - - - org.apache.maven.plugins - maven-remote-resources-plugin - ${maven-remote-resources-plugin.version} - - - process-resource-bundles - none - - - - - - - - org.apache.maven.plugins - maven-remote-resources-plugin - - - - + + This is performed from a script outside Maven apache-release @@ -624,44 +582,7 @@ under the License. - - - - java8only - - [1.8,1.9),8 - - - - - java11plus - - [11,14) - - - 8 - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - @{argLine} - --add-exports java.base/jdk.internal.misc=ALL-UNNAMED - --add-exports java.base/jdk.internal.ref=ALL-UNNAMED - --add-opens java.base/java.nio=ALL-UNNAMED - --add-opens java.base/sun.nio.ch=ALL-UNNAMED - - ${testng.generate-java-files},${testng.check-cpp-files} - - - - - - + --> generate-java-files diff --git a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java index 96c01d931..0998223e8 100644 --- a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java @@ -268,8 +268,7 @@ private WritableMemory growCombinedMemBuffer(final int itemSpaceNeeded) { final WritableMemory newMem = memReqSvr.request(mem_, needBytes); mem_.copyTo(0, newMem, 0, memBytes); - - memReqSvr.requestClose(mem_, newMem); + if (mem_.isCloseable()) { memReqSvr.requestClose(mem_, newMem); } return newMem; } diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java index 7a0fbb7ff..81f9e0d2e 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java @@ -57,8 +57,7 @@ public void createNewFilterTest() throws Exception { assertFalse(bf1.isReadOnly()); WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(sizeBytes, - new DefaultMemoryRequestServer())).scope()) { + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(sizeBytes)).scope()) { final BloomFilter bf2 = new BloomFilter(numBits, numHashes, seed, wmem); assertTrue(bf2.isEmpty()); diff --git a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java index c6c006f29..985cdf798 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java @@ -73,8 +73,7 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool //println("DIRECT"); byte[] barr1; WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes, - new DefaultMemoryRequestServer())).scope()) { + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes)).scope()) { //byte[] byteArr = new byte[bytes]; //WritableMemory wmem = WritableMemory.wrap(byteArr); hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); diff --git a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java index d19d3b471..b1bd5818d 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java @@ -67,8 +67,7 @@ public void test() { DoublesUnion dUnion; DoublesSketch dSketch; WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(10_000_000, - new DefaultMemoryRequestServer())).scope()) { + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(10_000_000)).scope()) { dUnion = DoublesUnion.builder().setMaxK(8).build(wmem); for (int s = 0; s < numSketches; s++) { dUnion.union(sketchArr[s]); } dSketch = dUnion.getResult(); //result is on heap diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java index 1ad55696c..1e2089b51 100644 --- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java @@ -84,8 +84,7 @@ public void checkHeapifyWrap(int k, int u, boolean ordered) { int bytes = usk.getCompactBytes(); //for Compact WritableMemory directMem; - try (ResourceScope scope = (directMem = WritableMemory.allocateDirect(bytes, - new DefaultMemoryRequestServer())).scope()) { + try (ResourceScope scope = (directMem = WritableMemory.allocateDirect(bytes)).scope()) { /**Via CompactSketch.compact**/ refSk = usk.compact(ordered, directMem); diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java index 32b9bbb44..63957c3cd 100644 --- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java @@ -683,8 +683,7 @@ public void checkEstModeNativeMemory() { int memCapacity = (k << 4) + (Family.QUICKSELECT.getMinPreLongs() << 3); WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(memCapacity, - new DefaultMemoryRequestServer())).scope()) { + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(memCapacity)).scope()) { UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(wmem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks @@ -930,7 +929,7 @@ private static final int getMaxBytes(int k) { } private static WritableMemory makeNativeMemory(int k) { - return WritableMemory.allocateDirect(getMaxBytes(k), new DefaultMemoryRequestServer()); + return WritableMemory.allocateDirect(getMaxBytes(k)); } } diff --git a/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java b/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java index d3617c67e..6c90eafe4 100644 --- a/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java +++ b/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java @@ -566,7 +566,7 @@ public void checkWrapSketchGivenDifferentSeed() { private static WritableMemory putOffHeap(Memory heapMem) { final long cap = heapMem.getCapacity(); - WritableMemory wmem = WritableMemory.allocateDirect(cap, new DefaultMemoryRequestServer()); + WritableMemory wmem = WritableMemory.allocateDirect(cap); heapMem.copyTo(0, wmem, 0, cap); return wmem; } diff --git a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java index fe76240a7..f28212061 100644 --- a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java +++ b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java @@ -234,8 +234,7 @@ public void checkUnionCompactOrderedSource() { final int bytes = Sketches.getCompactSketchMaxBytes(lgK); WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes, - new DefaultMemoryRequestServer())).scope()) { + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes)).scope()) { final CompactSketch csk = sk.compact(true, wmem); //ordered, direct final Union union = Sketches.setOperationBuilder().buildUnion(); From 59a3ff46fe966c98466ffb9f2b1925245952f38d Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 9 Oct 2024 14:25:35 -0700 Subject: [PATCH 15/88] Add kll main 8 longs --- .../apache/datasketches/kll/KllSketch.java | 9 +- .../kll/KllDirectCompactLongsSketchTest.java | 224 +++++ .../kll/KllDirectLongsSketchIteratorTest.java | 78 ++ .../kll/KllDirectLongsSketchTest.java | 686 +++++++++++++++ .../kll/KllLongsSketchIteratorTest.java | 114 +++ .../kll/KllLongsSketchSerDeTest.java | 131 +++ .../datasketches/kll/KllLongsSketchTest.java | 719 ++++++++++++++++ .../kll/KllMiscDirectLongsTest.java | 459 ++++++++++ .../datasketches/kll/KllMiscLongsTest.java | 790 ++++++++++++++++++ 9 files changed, 3209 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/apache/datasketches/kll/KllDirectCompactLongsSketchTest.java create mode 100644 src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchIteratorTest.java create mode 100644 src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java create mode 100644 src/test/java/org/apache/datasketches/kll/KllLongsSketchIteratorTest.java create mode 100644 src/test/java/org/apache/datasketches/kll/KllLongsSketchSerDeTest.java create mode 100644 src/test/java/org/apache/datasketches/kll/KllLongsSketchTest.java create mode 100644 src/test/java/org/apache/datasketches/kll/KllMiscDirectLongsTest.java create mode 100644 src/test/java/org/apache/datasketches/kll/KllMiscLongsTest.java diff --git a/src/main/java/org/apache/datasketches/kll/KllSketch.java b/src/main/java/org/apache/datasketches/kll/KllSketch.java index c398ed8ce..03b3a2bee 100644 --- a/src/main/java/org/apache/datasketches/kll/KllSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllSketch.java @@ -34,6 +34,7 @@ import static org.apache.datasketches.kll.KllSketch.SketchType.DOUBLES_SKETCH; import static org.apache.datasketches.kll.KllSketch.SketchType.FLOATS_SKETCH; import static org.apache.datasketches.kll.KllSketch.SketchType.ITEMS_SKETCH; +import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH; import java.util.Arrays; import java.util.Random; @@ -450,6 +451,8 @@ final boolean isCompactSingleItem() { boolean isFloatsSketch() { return sketchType == FLOATS_SKETCH; } + boolean isLongsSketch() { return sketchType == LONGS_SKETCH; } + boolean isItemsSketch() { return sketchType == ITEMS_SKETCH; } abstract boolean isLevelZeroSorted(); @@ -503,7 +506,11 @@ public enum SketchType { /** * KllItemsSketch */ - ITEMS_SKETCH(0, "KllItemsSketch"); + ITEMS_SKETCH(0, "KllItemsSketch"), + /** + * KllDoublesSketch + */ + LONGS_SKETCH(Long.BYTES, "KllLongsSketch"); private int typeBytes; private String name; diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectCompactLongsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectCompactLongsSketchTest.java new file mode 100644 index 000000000..6b57fccac --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllDirectCompactLongsSketchTest.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.kll.KllDirectLongsSketch.KllDirectCompactLongsSketch; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.WritableMemory; +import org.testng.annotations.Test; + +public class KllDirectCompactLongsSketchTest { + private static final DefaultMemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); + + @Test + public void checkRODirectUpdatable_ROandWritable() { + int k = 20; + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= k + 1; i++) { sk.update(i); } + byte[] byteArr = KllHelper.toByteArray(sk, true); //request updatable + Memory srcMem = Memory.wrap(byteArr); //cast to Memory -> read only + KllLongsSketch sk2 = KllLongsSketch.wrap(srcMem); + assertTrue(sk2 instanceof KllDirectLongsSketch); + + assertTrue(sk2.isMemoryUpdatableFormat()); + assertTrue(sk2.isReadOnly()); + assertEquals(sk2.getMinItem(), 1L); + assertEquals(sk2.getMaxItem(), 21L); + + WritableMemory srcWmem = WritableMemory.writableWrap(byteArr); + KllLongsSketch sk3 = KllLongsSketch.writableWrap(srcWmem, memReqSvr); + assertTrue(sk3 instanceof KllDirectLongsSketch); + println(sk3.toString(true, false)); + assertFalse(sk3.isReadOnly()); + sk3.update(22); + assertEquals(sk2.getMinItem(), 1L); + assertEquals(sk2.getMaxItem(), 22L); + } + + @Test + public void checkRODirectCompact() { + int k = 20; + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= k + 1; i++) { sk.update(i); } + Memory srcMem = Memory.wrap(sk.toByteArray()); //compact RO fmt + KllLongsSketch sk2 = KllLongsSketch.wrap(srcMem); + assertTrue(sk2 instanceof KllDirectCompactLongsSketch); + //println(sk2.toString(true, false)); + assertFalse(sk2.isMemoryUpdatableFormat()); + assertTrue(sk2.isReadOnly()); + assertEquals(sk2.getMinItem(), 1L); + assertEquals(sk2.getMaxItem(), 21L); + Memory srcMem2 = Memory.wrap(sk2.toByteArray()); + KllLongsSketch sk3 = KllLongsSketch.writableWrap((WritableMemory)srcMem2, memReqSvr); + assertTrue(sk3 instanceof KllDirectCompactLongsSketch); + assertFalse(sk2.isMemoryUpdatableFormat()); + //println(sk3.toString(true, false)); + assertTrue(sk3.isReadOnly()); + assertEquals(sk3.getMinItem(), 1L); + assertEquals(sk3.getMaxItem(), 21L); + } + + @Test + public void checkDirectCompactSingleItem() { + int k = 20; + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + + sk.update(1); + KllLongsSketch sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray())); + assertTrue(sk2 instanceof KllDirectCompactLongsSketch); + //println(sk2.toString(true, false)); + assertTrue(sk2.isReadOnly()); + assertEquals(sk2.getLongSingleItem(), 1L); + + sk.update(2); + sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray())); + assertEquals(sk2.getN(), 2); + try { + sk2.getLongSingleItem(); + fail(); + } catch (SketchesArgumentException e) { } + } + + @Test + public void checkDirectCompactGetLongItemsArray() { + int k = 20; + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + + KllLongsSketch sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray())); + long[] itemsArr = sk2.getLongItemsArray(); + for (int i = 0; i < 20; i++) { assertEquals(itemsArr[i], 0); } + + sk.update(1); + sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray())); + itemsArr = sk2.getLongItemsArray(); + for (int i = 0; i < 19; i++) { assertEquals(itemsArr[i], 0); } + assertEquals(itemsArr[19], 1L); + + for (int i = 2; i <= 21; i++) { sk.update(i); } + sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray())); + itemsArr = sk2.getLongItemsArray(); + assertEquals(itemsArr.length, 33); + assertEquals(itemsArr[22], 21); + } + + @Test + public void checkHeapAndDirectCompactGetRetainedItemsArray() { + int k = 20; + + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + long[] retArr = sk.getLongRetainedItemsArray(); + assertEquals(retArr.length, 0); + + KllLongsSketch sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray())); + retArr = sk2.getLongRetainedItemsArray(); + assertEquals(retArr.length, sk.getNumRetained()); + assertEquals(retArr.length, 0); + + sk.update(1); + retArr = sk.getLongRetainedItemsArray(); + assertEquals(retArr.length, sk.getNumRetained()); + assertEquals(retArr.length, 1); + assertEquals(retArr[0], 1L); + + sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray())); + retArr = sk2.getLongRetainedItemsArray(); + assertEquals(retArr.length, sk.getNumRetained()); + assertEquals(retArr.length, 1); + assertEquals(retArr[0], 1L); + + for (int i = 2; i <= 21; i++) { sk.update(i); } + retArr = sk.getLongRetainedItemsArray(); + assertEquals(retArr.length, sk.getNumRetained()); + assertEquals(retArr.length, 11); + + sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray())); + assertEquals(retArr.length, sk2.getNumRetained()); + assertEquals(retArr.length, 11); + } + + @Test + public void checkMinAndMax() { + int k = 20; + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + KllLongsSketch sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray())); + try { sk2.getMinItem(); fail(); } catch (SketchesArgumentException e) {} + try { sk2.getMaxItem(); fail(); } catch (SketchesArgumentException e) {} + sk.update(1); + sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray())); + assertEquals(sk2.getMaxItem(),1L); + assertEquals(sk2.getMinItem(),1L); + for (int i = 2; i <= 21; i++) { sk.update(i); } + sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray())); + assertEquals(sk2.getMaxItem(),21L); + assertEquals(sk2.getMinItem(),1L); + } + + @Test + public void checkQuantile() { + KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(); + for (int i = 1; i <= 1000; i++) { sk1.update(i); } + KllLongsSketch sk2 = KllLongsSketch.wrap(Memory.wrap(sk1.toByteArray())); + long med2 = sk2.getQuantile(0.5); + long med1 = sk1.getQuantile(0.5); + assertEquals(med1, med2); + println("Med1: " + med1); + println("Med2: " + med2); + } + + @Test + public void checkCompactSingleItemMerge() { + int k = 20; + KllLongsSketch skH1 = KllLongsSketch.newHeapInstance(k); //Heap with 1 (single) + skH1.update(21); + KllLongsSketch skDC1 = KllLongsSketch.wrap(Memory.wrap(skH1.toByteArray())); //Direct Compact with 1 (single) + KllLongsSketch skH20 = KllLongsSketch.newHeapInstance(k); //Heap with 20 + for (int i = 1; i <= 20; i++) { skH20.update(i); } + skH20.merge(skDC1); + assertEquals(skH20.getN(), 21); + + WritableMemory wmem = WritableMemory.allocate(1000); + KllLongsSketch skDU20 = KllLongsSketch.newDirectInstance(k, wmem, memReqSvr);//Direct Updatable with 21 + for (int i = 1; i <= 20; i++) { skDU20.update(i); } + skDU20.merge(skDC1); + assertEquals(skDU20.getN(), 21); + } + + @Test + public void printlnTest() { + println("PRINTING: " + this.getClass().getName()); + } + + private final static boolean enablePrinting = false; + + /** + * @param o the Object to println + */ + private static final void println(final Object o) { + if (enablePrinting) { System.out.println(o.toString()); } + } + +} diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchIteratorTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchIteratorTest.java new file mode 100644 index 000000000..8be509f10 --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchIteratorTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import org.apache.datasketches.memory.DefaultMemoryRequestServer; +import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.quantilescommon.QuantilesLongsSketchIterator; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class KllDirectLongsSketchIteratorTest { + private static final DefaultMemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); + + @Test + public void emptySketch() { + final KllLongsSketch sketch = getDLSketch(200, 0); + QuantilesLongsSketchIterator it = sketch.iterator(); + Assert.assertFalse(it.next()); + } + + @Test + public void oneItemSketch() { + final KllLongsSketch sketch = getDLSketch(200, 0); + sketch.update(0); + QuantilesLongsSketchIterator it = sketch.iterator(); + Assert.assertTrue(it.next()); + Assert.assertEquals(it.getQuantile(), 0); + Assert.assertEquals(it.getWeight(), 1); + Assert.assertFalse(it.next()); + } + + @Test + public void bigSketches() { + for (int n = 1000; n < 100000; n += 2000) { + final KllLongsSketch sketch = getDLSketch(200, 0); + for (int i = 0; i < n; i++) { + sketch.update(i); + } + QuantilesLongsSketchIterator it = sketch.iterator(); + int count = 0; + int weight = 0; + while (it.next()) { + count++; + weight += (int)it.getWeight(); + } + Assert.assertEquals(count, sketch.getNumRetained()); + Assert.assertEquals(weight, n); + } + } + + private static KllLongsSketch getDLSketch(final int k, final int n) { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= n; i++) { sk.update(i); } + byte[] byteArr = KllHelper.toByteArray(sk, true); + WritableMemory wmem = WritableMemory.writableWrap(byteArr); + + KllLongsSketch dlsk = KllLongsSketch.writableWrap(wmem, memReqSvr); + return dlsk; + } + +} diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java new file mode 100644 index 000000000..f1784b7ce --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java @@ -0,0 +1,686 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.kll.KllSketch.SketchStructure; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.WritableMemory; +import org.testng.annotations.Test; + +import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH; +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.EXCLUSIVE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +public class KllDirectLongsSketchTest { + + private static final double PMF_EPS_FOR_K_8 = 0.35; // PMF rank error (epsilon) for k=8 + private static final double PMF_EPS_FOR_K_128 = 0.025; // PMF rank error (epsilon) for k=128 + private static final double PMF_EPS_FOR_K_256 = 0.013; // PMF rank error (epsilon) for k=256 + private static final double NUMERIC_NOISE_TOLERANCE = 1E-6; + private static final DefaultMemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); + + @Test + public void empty() { + final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0); + assertTrue(sketch.isEmpty()); + assertEquals(sketch.getN(), 0); + assertEquals(sketch.getNumRetained(), 0); + try { sketch.getRank(0); fail(); } catch (SketchesArgumentException e) {} + try { sketch.getMinItem(); fail(); } catch (SketchesArgumentException e) {} + try { sketch.getMaxItem(); fail(); } catch (SketchesArgumentException e) {} + try { sketch.getQuantile(0.5); fail(); } catch (SketchesArgumentException e) {} + try { sketch.getQuantiles(new double[] {0.0, 1.0}); fail(); } catch (SketchesArgumentException e) {} + try { sketch.getPMF(new long[] {0}); fail(); } catch (SketchesArgumentException e) {} + try { sketch.getCDF(new long[0]); fail(); } catch (SketchesArgumentException e) {} + assertNotNull(sketch.toString(true, true)); + assertNotNull(sketch.toString()); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void getQuantileInvalidArg() { + final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0); + sketch.update(1); + sketch.getQuantile(-1.0); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void getQuantilesInvalidArg() { + final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0); + sketch.update(1); + sketch.getQuantiles(new double[] {2.0}); + } + + @Test + public void oneValue() { + final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0); + sketch.update(1); + assertFalse(sketch.isEmpty()); + assertEquals(sketch.getN(), 1); + assertEquals(sketch.getNumRetained(), 1); + assertEquals(sketch.getRank(1, EXCLUSIVE), 0.0); + assertEquals(sketch.getRank(2, EXCLUSIVE), 1.0); + assertEquals(sketch.getMinItem(), 1L); + assertEquals(sketch.getMaxItem(), 1L); + assertEquals(sketch.getQuantile(0.5, EXCLUSIVE), 1L); + } + + @Test + public void manyValuesEstimationMode() { + final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0); + final int n = 1_000_000; + + for (int i = 0; i < n; i++) { + sketch.update(i); + } + assertEquals(sketch.getN(), n); + + // test getRank + for (int i = 0; i < n; i++) { + final double trueRank = (double) i / n; + assertEquals(sketch.getRank(i), trueRank, PMF_EPS_FOR_K_256, "for value " + i); + } + + // test getPMF + final double[] pmf = sketch.getPMF(new long[] {n / 2}); // split at median + assertEquals(pmf.length, 2); + assertEquals(pmf[0], 0.5, PMF_EPS_FOR_K_256); + assertEquals(pmf[1], 0.5, PMF_EPS_FOR_K_256); + assertEquals(sketch.getMinItem(), 0); // min value is exact + assertEquals(sketch.getMaxItem(), n - 1L); // max value is exact + + // check at every 0.1 percentage point + final double[] ranks = new double[1001]; + final double[] reverseRanks = new double[1001]; // check that ordering doesn't matter + for (int i = 0; i <= 1000; i++) { + ranks[i] = (double) i / 1000; + reverseRanks[1000 - i] = ranks[i]; + } + final long[] quantiles = sketch.getQuantiles(ranks); + final long[] reverseQuantiles = sketch.getQuantiles(reverseRanks); + long previousQuantile = 0; + for (int i = 0; i <= 1000; i++) { + final long quantile = sketch.getQuantile(ranks[i]); + assertEquals(quantile, quantiles[i]); + assertEquals(quantile, reverseQuantiles[1000 - i]); + assertTrue(previousQuantile <= quantile); + previousQuantile = quantile; + } + } + + @Test + public void getRankGetCdfGetPmfConsistency() { + final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0); + final int n = 1000; + final long[] values = new long[n]; + for (int i = 0; i < n; i++) { + sketch.update(i); + values[i] = i; + } + final double[] ranks = sketch.getCDF(values); + final double[] pmf = sketch.getPMF(values); + double sumPmf = 0; + for (int i = 0; i < n; i++) { + assertEquals(ranks[i], sketch.getRank(values[i]), NUMERIC_NOISE_TOLERANCE, + "rank vs CDF for value " + i); + sumPmf += pmf[i]; + assertEquals(ranks[i], sumPmf, NUMERIC_NOISE_TOLERANCE, "CDF vs PMF for value " + i); + } + sumPmf += pmf[n]; + assertEquals(sumPmf, 1.0, NUMERIC_NOISE_TOLERANCE); + assertEquals(ranks[n], 1.0, NUMERIC_NOISE_TOLERANCE); + } + + @Test + public void merge() { + final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0); + final KllLongsSketch sketch2 = getUpdatableDirectLongSketch(200, 0); + final int n = 10_000; + for (int i = 0; i < n; i++) { + sketch1.update(i); + sketch2.update(2 * n - i - 1); + } + + assertEquals(sketch1.getMinItem(), 0); + assertEquals(sketch1.getMaxItem(), n - 1); + + assertEquals(sketch2.getMinItem(), n); + assertEquals(sketch2.getMaxItem(), 2 * n - 1); + + sketch1.merge(sketch2); + + assertFalse(sketch1.isEmpty()); + assertEquals(sketch1.getN(), 2L * n); + assertEquals(sketch1.getMinItem(), 0); + assertEquals(sketch1.getMaxItem(), 2 * n - 1L); + assertEquals(sketch1.getQuantile(0.5), n, n * PMF_EPS_FOR_K_256); + } + + @Test + public void mergeLowerK() { + final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(256, 0); + final KllLongsSketch sketch2 = getUpdatableDirectLongSketch(128, 0); + final int n = 10_000; + for (int i = 0; i < n; i++) { + sketch1.update(i); + sketch2.update(2 * n - i - 1); + } + + assertEquals(sketch1.getMinItem(), 0); + assertEquals(sketch1.getMaxItem(), n - 1); + + assertEquals(sketch2.getMinItem(), n); + assertEquals(sketch2.getMaxItem(), 2 * n - 1); + + assertTrue(sketch1.getNormalizedRankError(false) < sketch2.getNormalizedRankError(false)); + assertTrue(sketch1.getNormalizedRankError(true) < sketch2.getNormalizedRankError(true)); + sketch1.merge(sketch2); + + // sketch1 must get "contaminated" by the lower K in sketch2 + assertEquals(sketch1.getNormalizedRankError(false), sketch2.getNormalizedRankError(false)); + assertEquals(sketch1.getNormalizedRankError(true), sketch2.getNormalizedRankError(true)); + + assertFalse(sketch1.isEmpty()); + assertEquals(sketch1.getN(), 2 * n); + assertEquals(sketch1.getMinItem(), 0); + assertEquals(sketch1.getMaxItem(), 2 * n - 1); + assertEquals(sketch1.getQuantile(0.5), n, n * PMF_EPS_FOR_K_128); + } + + @Test + public void mergeEmptyLowerK() { + final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(256, 0); + final KllLongsSketch sketch2 = getUpdatableDirectLongSketch(128, 0); + final int n = 10_000; + for (int i = 0; i < n; i++) { + sketch1.update(i); + } + + // rank error should not be affected by a merge with an empty sketch with lower K + final double rankErrorBeforeMerge = sketch1.getNormalizedRankError(true); + sketch1.merge(sketch2); + assertEquals(sketch1.getNormalizedRankError(true), rankErrorBeforeMerge); + + assertFalse(sketch1.isEmpty()); + assertEquals(sketch1.getN(), n); + assertEquals(sketch1.getMinItem(), 0); + assertEquals(sketch1.getMaxItem(), n - 1); + assertEquals(sketch1.getQuantile(0.5), n / 2, n / 2 * PMF_EPS_FOR_K_256); + + //merge the other way + sketch2.merge(sketch1); + assertFalse(sketch1.isEmpty()); + assertEquals(sketch1.getN(), n); + assertEquals(sketch1.getMinItem(), 0); + assertEquals(sketch1.getMaxItem(), n - 1); + assertEquals(sketch1.getQuantile(0.5), n / 2, n / 2 * PMF_EPS_FOR_K_256); + } + + @Test + public void mergeExactModeLowerK() { + final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(256, 0); + final KllLongsSketch sketch2 = getUpdatableDirectLongSketch(128, 0); + final int n = 10000; + for (int i = 0; i < n; i++) { + sketch1.update(i); + } + sketch2.update(1); + + // rank error should not be affected by a merge with a sketch in exact mode with lower K + final double rankErrorBeforeMerge = sketch1.getNormalizedRankError(true); + sketch1.merge(sketch2); + assertEquals(sketch1.getNormalizedRankError(true), rankErrorBeforeMerge); + } + + @Test + public void mergeMinMinValueFromOther() { + final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0); + final KllLongsSketch sketch2 = getUpdatableDirectLongSketch(200, 0); + sketch1.update(1); + sketch2.update(2); + sketch2.merge(sketch1); + assertEquals(sketch2.getMinItem(), 1L); + } + + @Test + public void mergeMinAndMaxFromOther() { + final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0); + final KllLongsSketch sketch2 = getUpdatableDirectLongSketch(200, 0); + int n = 1_000_000; + for (int i = 1; i <= n; i++) { + sketch1.update(i); + } + sketch2.merge(sketch1); + assertEquals(sketch2.getMinItem(), 1L); + assertEquals(sketch2.getMaxItem(), 1_000_000L); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void kTooSmall() { + getUpdatableDirectLongSketch(KllSketch.DEFAULT_M - 1, 0); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void kTooLarge() { + getUpdatableDirectLongSketch(KllSketch.MAX_K + 1, 0); + } + + @Test + public void minK() { + final KllLongsSketch sketch = getUpdatableDirectLongSketch(KllSketch.DEFAULT_M, 0); + for (int i = 0; i < 1000; i++) { + sketch.update(i); + } + assertEquals(sketch.getK(), KllSketch.DEFAULT_M); + assertEquals(sketch.getQuantile(0.5), 500, 500 * PMF_EPS_FOR_K_8); + } + + @Test + public void maxK() { + final KllLongsSketch sketch = getUpdatableDirectLongSketch(KllSketch.MAX_K, 0); + for (int i = 0; i < 1000; i++) { + sketch.update(i); + } + assertEquals(sketch.getK(), KllSketch.MAX_K); + assertEquals(sketch.getQuantile(0.5), 500, 500 * PMF_EPS_FOR_K_256); + } + + @Test + public void serializeDeserializeEmptyViaCompactHeapify() { + final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0); + final byte[] bytes = sketch1.toByteArray(); //compact + final KllLongsSketch sketch2 = KllLongsSketch.heapify(Memory.wrap(bytes)); + assertEquals(bytes.length, sketch1.currentSerializedSizeBytes(false)); + assertTrue(sketch2.isEmpty()); + assertEquals(sketch2.getNumRetained(), sketch1.getNumRetained()); + assertEquals(sketch2.getN(), sketch1.getN()); + assertEquals(sketch2.getNormalizedRankError(false), sketch1.getNormalizedRankError(false)); + try { sketch2.getMinItem(); fail(); } catch (SketchesArgumentException e) {} + try { sketch2.getMaxItem(); fail(); } catch (SketchesArgumentException e) {} + assertEquals(sketch2.currentSerializedSizeBytes(false), + sketch1.currentSerializedSizeBytes(false)); + } + + @Test + public void serializeDeserializeEmptyViaUpdatableWritableWrap() { + final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0); + final byte[] bytes = KllHelper.toByteArray(sketch1, true); + final KllLongsSketch sketch2 = + KllLongsSketch.writableWrap(WritableMemory.writableWrap(bytes),memReqSvr); + assertEquals(bytes.length, sketch1.currentSerializedSizeBytes(true)); + assertTrue(sketch2.isEmpty()); + assertEquals(sketch2.getNumRetained(), sketch1.getNumRetained()); + assertEquals(sketch2.getN(), sketch1.getN()); + assertEquals(sketch2.getNormalizedRankError(false), sketch1.getNormalizedRankError(false)); + try { sketch2.getMinItem(); fail(); } catch (SketchesArgumentException e) {} + try { sketch2.getMaxItem(); fail(); } catch (SketchesArgumentException e) {} + assertEquals(sketch2.currentSerializedSizeBytes(true), + sketch1.currentSerializedSizeBytes(true)); + } + + @Test + public void serializeDeserializeOneValueViaCompactHeapify() { + final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0); + sketch1.update(1); + final byte[] bytes = sketch1.toByteArray(); + final KllLongsSketch sketch2 = KllLongsSketch.heapify(Memory.wrap(bytes)); + assertEquals(bytes.length, sketch1.currentSerializedSizeBytes(false)); + assertFalse(sketch2.isEmpty()); + assertEquals(sketch2.getNumRetained(), 1); + assertEquals(sketch2.getN(), 1); + assertEquals(sketch2.getNormalizedRankError(false), sketch1.getNormalizedRankError(false)); + assertTrue(sketch2.getMinItem() < Long.MAX_VALUE); + assertTrue(sketch2.getMaxItem() > Long.MIN_VALUE); + assertEquals(sketch2.currentSerializedSizeBytes(false), 8 + Long.BYTES); + } + + @Test + public void serializeDeserializeOneValueViaUpdatableWritableWrap() { + final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0); + sketch1.update(1); + final byte[] bytes = KllHelper.toByteArray(sketch1, true); + final KllLongsSketch sketch2 = + KllLongsSketch.writableWrap(WritableMemory.writableWrap(bytes),memReqSvr); + assertEquals(bytes.length, sketch1.currentSerializedSizeBytes(true)); + assertFalse(sketch2.isEmpty()); + assertEquals(sketch2.getNumRetained(), 1); + assertEquals(sketch2.getN(), 1); + assertEquals(sketch2.getNormalizedRankError(false), sketch1.getNormalizedRankError(false)); + assertEquals(sketch2.getMinItem(), 1L); + assertEquals(sketch2.getMaxItem(), 1L); + assertEquals(sketch2.currentSerializedSizeBytes(false), 8 + Long.BYTES); + assertEquals(sketch2.currentSerializedSizeBytes(true), bytes.length); + } + + @Test + public void serializeDeserializeFullViaCompactHeapify() { + final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 1000); + final byte[] byteArr1 = sketch1.toByteArray(); //compact + final KllLongsSketch sketch2 = KllLongsSketch.heapify(Memory.wrap(byteArr1)); + assertEquals(byteArr1.length, sketch1.currentSerializedSizeBytes(false)); + assertFalse(sketch2.isEmpty()); + assertEquals(sketch2.getNumRetained(), sketch1.getNumRetained()); + assertEquals(sketch2.getN(), sketch1.getN()); + assertEquals(sketch2.getNormalizedRankError(false), sketch1.getNormalizedRankError(false)); + assertEquals(sketch2.getMinItem(), sketch1.getMinItem()); + assertEquals(sketch2.getMaxItem(), sketch1.getMaxItem()); + assertEquals(sketch2.currentSerializedSizeBytes(false), sketch1.currentSerializedSizeBytes(false)); + } + + @Test + public void serializeDeserializeFullViaUpdatableWritableWrap() { + final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0); + final int n = 1000; + for (int i = 1; i <= n; i++) { + sketch1.update(i); + } + final byte[] bytes = KllHelper.toByteArray(sketch1, true); //updatable + final KllLongsSketch sketch2 = + KllLongsSketch.writableWrap(WritableMemory.writableWrap(bytes), memReqSvr); + assertEquals(bytes.length, sketch1.currentSerializedSizeBytes(true)); + assertFalse(sketch2.isEmpty()); + assertEquals(sketch2.getNumRetained(), sketch1.getNumRetained()); + assertEquals(sketch2.getN(), sketch1.getN()); + assertEquals(sketch2.getNormalizedRankError(false), sketch1.getNormalizedRankError(false)); + assertEquals(sketch2.getMinItem(), sketch1.getMinItem()); + assertEquals(sketch2.getMaxItem(), sketch1.getMaxItem()); + assertEquals(sketch2.currentSerializedSizeBytes(true), sketch1.currentSerializedSizeBytes(true)); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void outOfOrderSplitPoints() { + final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0); + sketch.update(0); + sketch.getCDF(new long[] {1, 0}); + } + + @Test + public void checkSimpleMergeDirect() { //used for troubleshooting + int k = 20; + int n1 = 21; + int n2 = 43; + KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(k); + KllLongsSketch sk2 = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= n1; i++) { + sk1.update(i); + } + for (int i = 1; i <= n2; i++) { + sk2.update(i + 100); + } + println("SK1:"); + println(sk1.toString(true, true)); + println("SK2:"); + println(sk2.toString(true, true)); + WritableMemory wmem1 = WritableMemory.writableWrap(KllHelper.toByteArray(sk1, true)); + WritableMemory wmem2 = WritableMemory.writableWrap(KllHelper.toByteArray(sk2, true)); + KllLongsSketch dsk1 = KllLongsSketch.writableWrap(wmem1, memReqSvr); + KllLongsSketch dsk2 = KllLongsSketch.writableWrap(wmem2, memReqSvr); + println("BEFORE MERGE"); + println(dsk1.toString(true, true)); + dsk1.merge(dsk2); + println("AFTER MERGE"); + println(dsk1.toString(true, true)); + } + + @Test + public void checkSketchInitializeDirectLongUpdatableMem() { + int k = 20; //don't change this + KllLongsSketch sk; + KllLongsSketch sk2; + byte[] compBytes; + WritableMemory wmem; + + println("#### CASE: LONG FULL DIRECT FROM UPDATABLE"); + sk2 = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= k + 1; i++) { sk2.update(i); } + //println(sk2.toString(true, true)); + compBytes = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(compBytes); + println(KllPreambleUtil.toString(compBytes, LONGS_SKETCH, true)); + sk = KllLongsSketch.writableWrap(wmem, memReqSvr); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), k + 1); + assertEquals(sk.getNumRetained(), 11); + assertFalse(sk.isEmpty()); + assertTrue(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 33); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3); + assertEquals(sk.getMaxItem(), 21); + assertEquals(sk.getMinItem(), 1); + assertEquals(sk.getNumLevels(), 2); + assertFalse(sk.isLevelZeroSorted()); + + println("#### CASE: LONG EMPTY HEAPIFIED FROM UPDATABLE"); + sk2 = KllLongsSketch.newHeapInstance(k); + //println(sk.toString(true, true)); + compBytes = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(compBytes); + println(KllPreambleUtil.toString(compBytes, LONGS_SKETCH, true)); + sk = KllLongsSketch.writableWrap(wmem, memReqSvr); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumRetained(), 0); + assertTrue(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) {} + try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) {} + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + + println("#### CASE: LONG SINGLE HEAPIFIED FROM UPDATABLE"); + sk2 = KllLongsSketch.newHeapInstance(k); + sk2.update(1); + //println(sk.toString(true, true)); + compBytes = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(compBytes); + println(KllPreambleUtil.toString(compBytes, LONGS_SKETCH, true)); + sk = KllLongsSketch.writableWrap(wmem, memReqSvr); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 1); + assertEquals(sk.getNumRetained(), 1); + assertFalse(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + assertEquals(sk.getMaxItem(), 1L); + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + } + + @Test + public void checkGetWritableMemory() { + final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 200); + assertEquals(sketch.getK(), 200); + assertEquals(sketch.getN(), 200); + assertFalse(sketch.isEmpty()); + assertTrue(sketch.isMemoryUpdatableFormat()); + assertFalse(sketch.isEstimationMode()); + assertTrue(sketch.isLongsSketch()); + assertFalse(sketch.isLevelZeroSorted()); + assertFalse(sketch.isDoublesSketch()); + + final WritableMemory wmem = sketch.getWritableMemory(); + final KllLongsSketch sk = KllHeapLongsSketch.heapifyImpl(wmem); + assertEquals(sk.getK(), 200); + assertEquals(sk.getN(), 200); + assertFalse(sk.isEmpty()); + assertFalse(sk.isMemoryUpdatableFormat()); + assertFalse(sk.isEstimationMode()); + assertTrue(sk.isLongsSketch()); + assertFalse(sk.isLevelZeroSorted()); + assertFalse(sk.isDoublesSketch()); + } + + @Test + public void checkReset() { + WritableMemory dstMem = WritableMemory.allocate(3000); + KllLongsSketch sk = KllLongsSketch.newDirectInstance(20, dstMem, memReqSvr); + for (int i = 1; i <= 100; i++) { sk.update(i); } + long n1 = sk.getN(); + long min1 = sk.getMinItem(); + long max1 = sk.getMaxItem(); + sk.reset(); + for (int i = 1; i <= 100; i++) { sk.update(i); } + long n2 = sk.getN(); + long min2 = sk.getMinItem(); + long max2 = sk.getMaxItem(); + assertEquals(n2, n1); + assertEquals(min2, min1); + assertEquals(max2, max1); + } + + @Test + public void checkHeapify() { + WritableMemory dstMem = WritableMemory.allocate(6000); + KllLongsSketch sk = KllLongsSketch.newDirectInstance(20, dstMem, memReqSvr); + for (int i = 1; i <= 100; i++) { sk.update(i); } + KllLongsSketch sk2 = KllHeapLongsSketch.heapifyImpl(dstMem); + assertEquals(sk2.getMinItem(), 1L); + assertEquals(sk2.getMaxItem(), 100L); + } + + @Test + public void checkMergeKllLongsSketch() { + WritableMemory dstMem = WritableMemory.allocate(6000); + KllLongsSketch sk = KllLongsSketch.newDirectInstance(20, dstMem, memReqSvr); + for (int i = 1; i <= 21; i++) { sk.update(i); } + KllLongsSketch sk2 = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= 21; i++ ) { sk2.update(i + 100); } + sk.merge(sk2); + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getMaxItem(), 121L); + } + + @Test + public void checkReverseMergeKllLongsSketch() { + WritableMemory dstMem = WritableMemory.allocate(6000); + KllLongsSketch sk = KllLongsSketch.newDirectInstance(20, dstMem, memReqSvr); + for (int i = 1; i <= 21; i++) { sk.update(i); } + KllLongsSketch sk2 = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= 21; i++ ) { sk2.update(i + 100); } + sk2.merge(sk); + assertEquals(sk2.getMinItem(), 1L); + assertEquals(sk2.getMaxItem(), 121L); + } + + @Test + public void checkWritableWrapOfCompactForm() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= 21; i++ ) { sk.update(i); } + WritableMemory srcMem = WritableMemory.writableWrap(sk.toByteArray()); + KllLongsSketch sk2 = KllLongsSketch.writableWrap(srcMem, memReqSvr); + assertEquals(sk2.getMinItem(), 1L); + assertEquals(sk2.getMaxItem(), 21L); + } + + @Test + public void checkReadOnlyExceptions() { + int k = 20; + long[] fltArr = new long[0]; + long fltV = 1; + int idx = 1; + boolean bool = true; + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + KllLongsSketch sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray())); + try { sk2.incN(1); fail(); } catch (SketchesArgumentException e) { } + try { sk2.incNumLevels(); fail(); } catch (SketchesArgumentException e) { } + try { sk2.setLongItemsArray(fltArr); fail(); } catch (SketchesArgumentException e) { } + try { sk2.setLongItemsArrayAt(idx, fltV); fail(); } catch (SketchesArgumentException e) { } + try { sk2.setLevelZeroSorted(bool); fail(); } catch (SketchesArgumentException e) { } + try { sk2.setMaxItem(fltV); fail(); } catch (SketchesArgumentException e) { } + try { sk2.setMinItem(fltV); fail(); } catch (SketchesArgumentException e) { } + try { sk2.setMinK(idx); fail(); } catch (SketchesArgumentException e) { } + try { sk2.setN(idx); fail(); } catch (SketchesArgumentException e) { } + try { sk2.setNumLevels(idx); fail(); } catch (SketchesArgumentException e) { } + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void checkMergeExceptions() { + KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(20); + WritableMemory srcMem1 = WritableMemory.writableWrap(sk1.toByteArray()); + KllLongsSketch sk2 = KllLongsSketch.writableWrap(srcMem1, memReqSvr); + sk2.merge(sk1); + } + + @Test + public void checkVectorUpdate() { + WritableMemory dstMem = WritableMemory.allocate(6000); + KllLongsSketch sk = KllLongsSketch.newDirectInstance(20, dstMem, memReqSvr); + long[] v = new long[21]; + for (int i = 0; i < 21; i++) { v[i] = i + 1; } + sk.update(v, 0, 21); + println(sk.toString(true, true)); + int[] levelsArr = sk.getLevelsArray(SketchStructure.UPDATABLE); + assertEquals(levelsArr[0], 22); + long[] longsArr = sk.getLongItemsArray(); + assertEquals(longsArr[22], 21); + } + + @Test + public void checkWeightedUpdate() { + WritableMemory dstMem = WritableMemory.allocate(6000); + KllLongsSketch sk = KllLongsSketch.newDirectInstance(8, dstMem, memReqSvr); + for (int i = 0; i < 16; i++) { + sk.update(i + 1, 16); + } + println(sk.toString(true, true)); + assertEquals(sk.getN(), 256); + assertEquals(sk.getMaxItem(), 16L); + assertEquals(sk.getMinItem(), 1L); + } + + private static KllLongsSketch getUpdatableDirectLongSketch(int k, int n) { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= n; i++) { sk.update(i); } + byte[] byteArr = KllHelper.toByteArray(sk, true); + WritableMemory wmem = WritableMemory.writableWrap(byteArr); + KllLongsSketch dfsk = KllLongsSketch.writableWrap(wmem, memReqSvr); + return dfsk; + } + + @Test + public void checkMergeExceptionsWrongType() { + KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(20); + KllDoublesSketch sk2 = KllDoublesSketch.newHeapInstance(20); + try { sk1.merge(sk2); fail(); } catch (ClassCastException e) { } + try { sk2.merge(sk1); fail(); } catch (ClassCastException e) { } + } + + private final static boolean enablePrinting = false; + + /** + * @param o the Object to println + */ + private static final void println(final Object o) { + if (enablePrinting) { System.out.println(o.toString()); } + } + +} diff --git a/src/test/java/org/apache/datasketches/kll/KllLongsSketchIteratorTest.java b/src/test/java/org/apache/datasketches/kll/KllLongsSketchIteratorTest.java new file mode 100644 index 000000000..a98c32c9e --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllLongsSketchIteratorTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import org.apache.datasketches.quantilescommon.LongsSortedViewIterator; +import org.apache.datasketches.quantilescommon.QuantilesLongsSketchIterator; +import org.testng.Assert; +import org.testng.annotations.Test; + +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.EXCLUSIVE; +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class KllLongsSketchIteratorTest { + + @Test + public void emptySketch() { + KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + QuantilesLongsSketchIterator it = sketch.iterator(); + Assert.assertFalse(it.next()); + } + + @Test + public void oneItemSketch() { + KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + sketch.update(1); + QuantilesLongsSketchIterator it = sketch.iterator(); + Assert.assertTrue(it.next()); + Assert.assertEquals(it.getQuantile(), 1L); + Assert.assertEquals(it.getWeight(), 1); + Assert.assertFalse(it.next()); + } + + @Test + public void twoItemSketchForIterator() { + KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + sketch.update(1); + sketch.update(2); + QuantilesLongsSketchIterator itr = sketch.iterator(); + assertTrue(itr.next()); + + assertEquals(itr.getQuantile(), 2L); + assertEquals(itr.getWeight(), 1); + + assertTrue(itr.next()); + + assertEquals(itr.getQuantile(), 1L); + assertEquals(itr.getWeight(), 1); + } + + @Test + public void twoItemSketchForSortedViewIterator() { + KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + sketch.update(1); + sketch.update(2); + LongsSortedViewIterator itr = sketch.getSortedView().iterator(); + + assertTrue(itr.next()); + + assertEquals(itr.getQuantile(), 1L); + assertEquals(itr.getWeight(), 1); + assertEquals(itr.getNaturalRank(EXCLUSIVE), 0); + assertEquals(itr.getNaturalRank(INCLUSIVE), 1); + assertEquals(itr.getNormalizedRank(EXCLUSIVE), 0); + assertEquals(itr.getNormalizedRank(INCLUSIVE), 0.5); + + assertTrue(itr.next()); + + assertEquals(itr.getQuantile(), 2L); + assertEquals(itr.getWeight(), 1); + assertEquals(itr.getNaturalRank(EXCLUSIVE), 1); + assertEquals(itr.getNaturalRank(INCLUSIVE), 2); + assertEquals(itr.getNormalizedRank(EXCLUSIVE), 0.5); + assertEquals(itr.getNormalizedRank(INCLUSIVE), 1.0); + } + + @Test + public void bigSketches() { + for (int n = 1000; n < 100000; n += 2000) { + KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + for (int i = 0; i < n; i++) { + sketch.update(i); + } + QuantilesLongsSketchIterator it = sketch.iterator(); + int count = 0; + int weight = 0; + while (it.next()) { + count++; + weight += (int)it.getWeight(); + } + Assert.assertEquals(count, sketch.getNumRetained()); + Assert.assertEquals(weight, n); + } + } + +} diff --git a/src/test/java/org/apache/datasketches/kll/KllLongsSketchSerDeTest.java b/src/test/java/org/apache/datasketches/kll/KllLongsSketchSerDeTest.java new file mode 100644 index 000000000..b9b0f800d --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllLongsSketchSerDeTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.memory.Memory; +import org.testng.annotations.Test; + +public class KllLongsSketchSerDeTest { + + @Test + public void serializeDeserializeEmpty() { + final int N = 20; + + final KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(N); + //Empty: from heap -> byte[] -> heap + final byte[] bytes = sk1.toByteArray(); + final KllLongsSketch sk2 = KllLongsSketch.heapify(Memory.wrap(bytes)); + assertEquals(bytes.length, sk1.getSerializedSizeBytes()); + assertTrue(sk2.isEmpty()); + assertEquals(sk2.getNumRetained(), sk1.getNumRetained()); + assertEquals(sk2.getN(), sk1.getN()); + assertEquals(sk2.getNormalizedRankError(false), sk1.getNormalizedRankError(false)); + try { sk2.getMinItem(); fail(); } catch (SketchesArgumentException e) {} + try { sk2.getMaxItem(); fail(); } catch (SketchesArgumentException e) {} + assertEquals(sk2.getSerializedSizeBytes(), sk1.getSerializedSizeBytes()); + + //Empty: from heap -> byte[] -> off heap + final KllLongsSketch sk3 = KllLongsSketch.wrap(Memory.wrap(bytes)); + assertTrue(sk3.isEmpty()); + assertEquals(sk3.getNumRetained(), sk1.getNumRetained()); + assertEquals(sk3.getN(), sk1.getN()); + assertEquals(sk3.getNormalizedRankError(false), sk1.getNormalizedRankError(false)); + try { sk3.getMinItem(); fail(); } catch (SketchesArgumentException e) {} + try { sk3.getMaxItem(); fail(); } catch (SketchesArgumentException e) {} + assertEquals(sk3.getSerializedSizeBytes(), sk1.getSerializedSizeBytes()); + //from heap -> byte[] -> off heap -> byte[] -> compare byte[] + final byte[] bytes2 = sk3.toByteArray(); + assertEquals(bytes, bytes2); + } + + @Test + public void serializeDeserializeOneValue() { + final KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(); + sk1.update(1); + + //from heap -> byte[] -> heap + final byte[] bytes = sk1.toByteArray(); + final KllLongsSketch sk2 = KllLongsSketch.heapify(Memory.wrap(bytes)); + assertEquals(bytes.length, sk1.getSerializedSizeBytes()); + assertFalse(sk2.isEmpty()); + assertEquals(sk2.getNumRetained(), 1); + assertEquals(sk2.getN(), 1); + assertEquals(sk2.getNormalizedRankError(false), sk1.getNormalizedRankError(false)); + assertEquals(sk2.getMinItem(), 1L); + assertEquals(sk2.getMaxItem(), 1L); + assertEquals(sk2.getSerializedSizeBytes(), Long.BYTES + Long.BYTES); + + //from heap -> byte[] -> off heap + final KllLongsSketch sk3 = KllLongsSketch.wrap(Memory.wrap(bytes)); + assertFalse(sk3.isEmpty()); + assertEquals(sk3.getNumRetained(), 1); + assertEquals(sk3.getN(), 1); + assertEquals(sk3.getNormalizedRankError(false), sk1.getNormalizedRankError(false)); + assertEquals(sk3.getMinItem(), 1L); + assertEquals(sk3.getMaxItem(), 1L); + assertEquals(sk3.getSerializedSizeBytes(), sk1.getSerializedSizeBytes()); + //from heap -> byte[] -> off heap -> byte[] -> compare byte[] + final byte[] bytes2 = sk3.toByteArray(); + assertEquals(bytes, bytes2); + } + + @Test + public void serializeDeserializeMultipleValues() { + final KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(); + final int n = 1000; + for (int i = 0; i < n; i++) { + sk1.update(i); + } + assertEquals(sk1.getMinItem(), 0); + assertEquals(sk1.getMaxItem(), 999L); + + //from heap -> byte[] -> heap + final byte[] bytes = sk1.toByteArray(); + final KllLongsSketch sk2 = KllLongsSketch.heapify(Memory.wrap(bytes)); + assertEquals(bytes.length, sk1.getSerializedSizeBytes()); + assertFalse(sk2.isEmpty()); + assertEquals(sk2.getNumRetained(), sk1.getNumRetained()); + assertEquals(sk2.getN(), sk1.getN()); + assertEquals(sk2.getNormalizedRankError(false), sk1.getNormalizedRankError(false)); + assertEquals(sk2.getMinItem(), sk1.getMinItem()); + assertEquals(sk2.getMaxItem(), sk1.getMaxItem()); + assertEquals(sk2.getSerializedSizeBytes(), sk1.getSerializedSizeBytes()); + + //from heap -> byte[] -> off heap + final KllLongsSketch sk3 = KllLongsSketch.wrap(Memory.wrap(bytes)); + assertFalse(sk3.isEmpty()); + assertEquals(sk3.getNumRetained(), sk1.getNumRetained()); + assertEquals(sk3.getN(), sk1.getN()); + assertEquals(sk3.getNormalizedRankError(false), sk1.getNormalizedRankError(false)); + assertEquals(sk3.getMinItem(), sk1.getMinItem()); + assertEquals(sk3.getMaxItem(), sk1.getMaxItem()); + assertEquals(sk3.getSerializedSizeBytes(), sk1.getSerializedSizeBytes()); + //from heap -> byte[] -> off heap -> byte[] -> compare byte[] + final byte[] bytes2 = sk3.toByteArray(); + assertEquals(bytes, bytes2); + } + +} diff --git a/src/test/java/org/apache/datasketches/kll/KllLongsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllLongsSketchTest.java new file mode 100644 index 000000000..1e4c1004a --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllLongsSketchTest.java @@ -0,0 +1,719 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import static java.lang.Math.min; +import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH; +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.EXCLUSIVE; +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.quantilescommon.LongsSortedView; +import org.apache.datasketches.quantilescommon.LongsSortedViewIterator; +import org.testng.annotations.Test; + +public class KllLongsSketchTest { + private static final String LS = System.getProperty("line.separator"); + private static final double PMF_EPS_FOR_K_8 = KllSketch.getNormalizedRankError(8, true); + private static final double PMF_EPS_FOR_K_128 = KllSketch.getNormalizedRankError(128, true); + private static final double PMF_EPS_FOR_K_256 = KllSketch.getNormalizedRankError(256, true); + private static final double NUMERIC_NOISE_TOLERANCE = 1E-6; + private static final DefaultMemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); + + @Test + public void empty() { + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + assertTrue(sketch.isEmpty()); + assertEquals(sketch.getN(), 0); + assertEquals(sketch.getNumRetained(), 0); + try { sketch.getRank(0); fail(); } catch (SketchesArgumentException e) {} + try { sketch.getMinItem(); fail(); } catch (SketchesArgumentException e) {} + try { sketch.getMaxItem(); fail(); } catch (SketchesArgumentException e) {} + try { sketch.getQuantile(0.5); fail(); } catch (SketchesArgumentException e) {} + try { sketch.getQuantiles(new double[] {0}); fail(); } catch (SketchesArgumentException e) {} + try { sketch.getPMF(new long[] {0}); fail(); } catch (SketchesArgumentException e) {} + try { sketch.getCDF(new long[] {0}); fail(); } catch (SketchesArgumentException e) {} + assertNotNull(sketch.toString(true, true)); + assertNotNull(sketch.toString()); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void getQuantileInvalidArg() { + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + sketch.update(1); + sketch.getQuantile(-1.0); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void getQuantilesInvalidArg() { + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + sketch.update(1); + sketch.getQuantiles(new double[] {2.0}); + } + + @Test + public void oneValue() { + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + sketch.update(1); + assertFalse(sketch.isEmpty()); + assertEquals(sketch.getN(), 1); + assertEquals(sketch.getNumRetained(), 1); + assertEquals(sketch.getRank(0L, EXCLUSIVE), 0.0); + assertEquals(sketch.getRank(1L, EXCLUSIVE), 0.0); + assertEquals(sketch.getRank(2L, EXCLUSIVE), 1.0); + assertEquals(sketch.getRank(0L, INCLUSIVE), 0.0); + assertEquals(sketch.getRank(1L, INCLUSIVE), 1.0); + assertEquals(sketch.getRank(2L, INCLUSIVE), 1.0); + assertEquals(sketch.getMinItem(), 1L); + assertEquals(sketch.getMaxItem(), 1L); + assertEquals(sketch.getQuantile(0.5, EXCLUSIVE), 1L); + assertEquals(sketch.getQuantile(0.5, INCLUSIVE), 1L); + } + + @Test + public void tenValues() { + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= 10; i++) { sketch.update(i); } + assertFalse(sketch.isEmpty()); + assertEquals(sketch.getN(), 10); + assertEquals(sketch.getNumRetained(), 10); + for (int i = 1; i <= 10; i++) { + assertEquals(sketch.getRank(i, EXCLUSIVE), (i - 1) / 10.0); + assertEquals(sketch.getRank(i, INCLUSIVE), i / 10.0); + } + final long[] qArr = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + double[] rOut = sketch.getRanks(qArr); //inclusive + for (int i = 0; i < qArr.length; i++) { + assertEquals(rOut[i], (i + 1) / 10.0); + } + rOut = sketch.getRanks(qArr, EXCLUSIVE); //exclusive + for (int i = 0; i < qArr.length; i++) { + assertEquals(rOut[i], i / 10.0); + } + + for (int i = 0; i >= 10; i++) { + double rank = i/10.0; + double q = rank == 1.0 ? i : i + 1; + assertEquals(sketch.getQuantile(rank, EXCLUSIVE), q); + q = rank == 0 ? i + 1 : i; + assertEquals(sketch.getQuantile(rank, INCLUSIVE), q); + } + + { + // getQuantile() and getQuantiles() equivalence EXCLUSIVE + final long[] quantiles = + sketch.getQuantiles(new double[] {0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0}, EXCLUSIVE); + for (int i = 0; i <= 10; i++) { + assertEquals(sketch.getQuantile(i / 10.0, EXCLUSIVE), quantiles[i]); + } + } + { + // getQuantile() and getQuantiles() equivalence INCLUSIVE + final long[] quantiles = + sketch.getQuantiles(new double[] {0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0}, INCLUSIVE); + for (int i = 0; i <= 10; i++) { + assertEquals(sketch.getQuantile(i / 10.0, INCLUSIVE), quantiles[i]); + } + } + } + + @Test + public void manyValuesEstimationMode() { + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + final int n = 1_000_000; + + for (int i = 0; i < n; i++) { + sketch.update(i); + } + assertEquals(sketch.getN(), n); + + // test getRank + for (int i = 0; i < n; i++) { + final double trueRank = (double) i / n; + assertEquals(sketch.getRank(i), trueRank, PMF_EPS_FOR_K_256, "for value " + i); + } + + // test getPMF + final double[] pmf = sketch.getPMF(new long[] {n / 2}); // split at median + assertEquals(pmf.length, 2); + assertEquals(pmf[0], 0.5, PMF_EPS_FOR_K_256); + assertEquals(pmf[1], 0.5, PMF_EPS_FOR_K_256); + + assertEquals(sketch.getMinItem(), 0); // min value is exact + assertEquals(sketch.getMaxItem(), n - 1); // max value is exact + + // check at every 0.1 percentage point + final double[] fractions = new double[1001]; + final double[] reverseFractions = new double[1001]; // check that ordering doesn't matter + for (int i = 0; i <= 1000; i++) { + fractions[i] = (double) i / 1000; + reverseFractions[1000 - i] = fractions[i]; + } + final long[] quantiles = sketch.getQuantiles(fractions); + final long[] reverseQuantiles = sketch.getQuantiles(reverseFractions); + double previousQuantile = 0; + for (int i = 0; i <= 1000; i++) { + final double quantile = sketch.getQuantile(fractions[i]); + assertEquals(quantile, quantiles[i]); + assertEquals(quantile, reverseQuantiles[1000 - i]); + assertTrue(previousQuantile <= quantile); + previousQuantile = quantile; + } + } + + @Test + public void getRankGetCdfGetPmfConsistency() { + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + final int n = 1000; + final long[] values = new long[n]; + for (int i = 0; i < n; i++) { + sketch.update(i); + values[i] = i; + } + { // inclusive = false (default) + final double[] ranks = sketch.getCDF(values); + final double[] pmf = sketch.getPMF(values); + double sumPmf = 0; + for (int i = 0; i < n; i++) { + assertEquals(ranks[i], sketch.getRank(values[i]), NUMERIC_NOISE_TOLERANCE, + "rank vs CDF for value " + i); + sumPmf += pmf[i]; + assertEquals(ranks[i], sumPmf, NUMERIC_NOISE_TOLERANCE, "CDF vs PMF for value " + i); + } + sumPmf += pmf[n]; + assertEquals(sumPmf, 1.0, NUMERIC_NOISE_TOLERANCE); + assertEquals(ranks[n], 1.0, NUMERIC_NOISE_TOLERANCE); + } + { // inclusive = true + final double[] ranks = sketch.getCDF(values, INCLUSIVE); + final double[] pmf = sketch.getPMF(values, INCLUSIVE); + double sumPmf = 0; + for (int i = 0; i < n; i++) { + assertEquals(ranks[i], sketch.getRank(values[i], INCLUSIVE), NUMERIC_NOISE_TOLERANCE, + "rank vs CDF for value " + i); + sumPmf += pmf[i]; + assertEquals(ranks[i], sumPmf, NUMERIC_NOISE_TOLERANCE, "CDF vs PMF for value " + i); + } + sumPmf += pmf[n]; + assertEquals(sumPmf, 1.0, NUMERIC_NOISE_TOLERANCE); + assertEquals(ranks[n], 1.0, NUMERIC_NOISE_TOLERANCE); + } + } + + @Test + public void merge() { + final KllLongsSketch sketch1 = KllLongsSketch.newHeapInstance(); + final KllLongsSketch sketch2 = KllLongsSketch.newHeapInstance(); + final int n = 10000; + for (int i = 0; i < n; i++) { + sketch1.update(i); + sketch2.update(2 * n - i - 1); + } + + assertEquals(sketch1.getMinItem(), 0); + assertEquals(sketch1.getMaxItem(), (n - 1)); + + assertEquals(sketch2.getMinItem(), n); + assertEquals(sketch2.getMaxItem(), (2 * n - 1)); + + sketch1.merge(sketch2); + + assertFalse(sketch1.isEmpty()); + assertEquals(sketch1.getN(), 2L * n); + assertEquals(sketch1.getMinItem(), 0); + assertEquals(sketch1.getMaxItem(), (2 * n - 1)); + assertEquals(sketch1.getQuantile(0.5), n, 2 * n * PMF_EPS_FOR_K_256); + } + + @Test + public void mergeLowerK() { + final KllLongsSketch sketch1 = KllLongsSketch.newHeapInstance(256); + final KllLongsSketch sketch2 = KllLongsSketch.newHeapInstance(128); + final int n = 10000; + for (int i = 0; i < n; i++) { + sketch1.update(i); + sketch2.update(2 * n - i - 1); + } + + assertEquals(sketch1.getMinItem(), 0); + assertEquals(sketch1.getMaxItem(), n - 1L); + + assertEquals(sketch2.getMinItem(), n); + assertEquals(sketch2.getMaxItem(), 2L * n - 1L); + + assertTrue(sketch1.getNormalizedRankError(false) < sketch2.getNormalizedRankError(false)); + assertTrue(sketch1.getNormalizedRankError(true) < sketch2.getNormalizedRankError(true)); + sketch1.merge(sketch2); + + // sketch1 must get "contaminated" by the lower K in sketch2 + assertEquals(sketch1.getNormalizedRankError(false), sketch2.getNormalizedRankError(false)); + assertEquals(sketch1.getNormalizedRankError(true), sketch2.getNormalizedRankError(true)); + + assertFalse(sketch1.isEmpty()); + assertEquals(sketch1.getN(), 2 * n); + assertEquals(sketch1.getMinItem(), 0); + assertEquals(sketch1.getMaxItem(), 2L * n - 1L); + assertEquals(sketch1.getQuantile(0.5), n, 2L * n * PMF_EPS_FOR_K_128); + } + + @Test + public void mergeEmptyLowerK() { + final KllLongsSketch sketch1 = KllLongsSketch.newHeapInstance(256); + final KllLongsSketch sketch2 = KllLongsSketch.newHeapInstance(128); + final int n = 10_000; + for (int i = 0; i < n; i++) { + sketch1.update(i); + } + + // rank error should not be affected by a merge with an empty sketch with lower K + final double rankErrorBeforeMerge = sketch1.getNormalizedRankError(true); + sketch1.merge(sketch2); + assertEquals(sketch1.getNormalizedRankError(true), rankErrorBeforeMerge); + + assertFalse(sketch1.isEmpty()); + assertEquals(sketch1.getN(), n); + assertEquals(sketch1.getMinItem(), 0); + assertEquals(sketch1.getMaxItem(), n - 1); + assertEquals(sketch1.getQuantile(0.5), n / 2, n * PMF_EPS_FOR_K_256); + + //merge the other way + sketch2.merge(sketch1); + assertFalse(sketch1.isEmpty()); + assertEquals(sketch1.getN(), n); + assertEquals(sketch1.getMinItem(), 0); + assertEquals(sketch1.getMaxItem(), n - 1); + assertEquals(sketch1.getQuantile(0.5), n / 2, n * PMF_EPS_FOR_K_256); + } + + @Test + public void mergeExactModeLowerK() { + final KllLongsSketch sketch1 = KllLongsSketch.newHeapInstance(256); + final KllLongsSketch sketch2 = KllLongsSketch.newHeapInstance(128); + final int n = 10000; + for (int i = 0; i < n; i++) { + sketch1.update(i); + } + sketch2.update(1); + + // rank error should not be affected by a merge with a sketch in exact mode with lower K + final double rankErrorBeforeMerge = sketch1.getNormalizedRankError(true); + sketch1.merge(sketch2); + assertEquals(sketch1.getNormalizedRankError(true), rankErrorBeforeMerge); + } + + @Test + public void mergeMinMinValueFromOther() { + final KllLongsSketch sketch1 = KllLongsSketch.newHeapInstance(); + final KllLongsSketch sketch2 = KllLongsSketch.newHeapInstance(); + sketch1.update(1); + sketch2.update(2); + sketch2.merge(sketch1); + assertEquals(sketch2.getMinItem(), 1); + } + + @Test + public void mergeMinAndMaxFromOther() { + final KllLongsSketch sketch1 = KllLongsSketch.newHeapInstance(); + for (int i = 1; i <= 1_000_000; i++) { + sketch1.update(i); + } + final KllLongsSketch sketch2 = KllLongsSketch.newHeapInstance(10); + sketch2.merge(sketch1); + assertEquals(sketch2.getMinItem(), 1); + assertEquals(sketch2.getMaxItem(), 1_000_000); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void kTooSmall() { + KllLongsSketch.newHeapInstance(KllSketch.DEFAULT_M - 1); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void kTooLarge() { + KllLongsSketch.newHeapInstance(KllSketch.MAX_K + 1); + } + + @Test + public void minK() { + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(KllSketch.DEFAULT_M); + for (int i = 0; i < 1000; i++) { + sketch.update(i); + } + assertEquals(sketch.getK(), KllSketch.DEFAULT_M); + assertEquals(sketch.getQuantile(0.5), 500, 1000 * PMF_EPS_FOR_K_8); + } + + @Test + public void maxK() { + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(KllSketch.MAX_K); + for (int i = 0; i < 1000; i++) { + sketch.update(i); + } + assertEquals(sketch.getK(), KllSketch.MAX_K); + assertEquals(sketch.getQuantile(0.5), 500, 1000 * PMF_EPS_FOR_K_256); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void outOfOrderSplitPoints() { + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + sketch.update(0); + sketch.getCDF(new long[] {1L, 0L}); + } + + @Test + public void checkReset() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= 100; i++) { sk.update(i); } + long n1 = sk.getN(); + double min1 = sk.getMinItem(); + double max1 = sk.getMaxItem(); + sk.reset(); + for (int i = 1; i <= 100; i++) { sk.update(i); } + long n2 = sk.getN(); + double min2 = sk.getMinItem(); + double max2 = sk.getMaxItem(); + assertEquals(n2, n1); + assertEquals(min2, min1); + assertEquals(max2, max1); + } + + @Test + public void checkReadOnlyUpdate() { + KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(20); + Memory mem = Memory.wrap(sk1.toByteArray()); + KllLongsSketch sk2 = KllLongsSketch.wrap(mem); + try { sk2.update(1); fail(); } catch (SketchesArgumentException e) { } + } + + @Test + public void checkNewDirectInstanceAndSize() { + WritableMemory wmem = WritableMemory.allocate(3000); + KllLongsSketch.newDirectInstance(wmem, memReqSvr); + try { KllLongsSketch.newDirectInstance(null, memReqSvr); fail(); } + catch (NullPointerException e) { } + try { KllLongsSketch.newDirectInstance(wmem, null); fail(); } + catch (NullPointerException e) { } + int updateSize = KllSketch.getMaxSerializedSizeBytes(200, 0, LONGS_SKETCH, true); + int compactSize = KllSketch.getMaxSerializedSizeBytes(200, 0, LONGS_SKETCH, false); + assertTrue(compactSize < updateSize); + } + + @Test + public void sortedView() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(); + sk.update(3); + sk.update(1); + sk.update(2); + + LongsSortedView view = sk.getSortedView(); + LongsSortedViewIterator itr = view.iterator(); + assertEquals(itr.next(), true); + assertEquals(itr.getQuantile(), 1); + assertEquals(itr.getWeight(), 1); + assertEquals(itr.getNaturalRank(EXCLUSIVE), 0); + assertEquals(itr.getNaturalRank(INCLUSIVE), 1); + assertEquals(itr.next(), true); + assertEquals(itr.getQuantile(), 2); + assertEquals(itr.getWeight(), 1); + assertEquals(itr.getNaturalRank(EXCLUSIVE), 1); + assertEquals(itr.getNaturalRank(INCLUSIVE), 2); + assertEquals(itr.next(), true); + assertEquals(itr.getQuantile(), 3); + assertEquals(itr.getWeight(), 1); + assertEquals(itr.getNaturalRank(EXCLUSIVE), 2); + assertEquals(itr.getNaturalRank(INCLUSIVE), 3); + assertEquals(itr.next(), false); + } + + @Test //also visual + public void checkCDF_PDF() { + final double[] cdfI = {.25, .50, .75, 1.0, 1.0 }; + final double[] cdfE = {0.0, .25, .50, .75, 1.0 }; + final double[] pmfI = {.25, .25, .25, .25, 0.0 }; + final double[] pmfE = {0.0, .25, .25, .25, .25 }; + final double toll = 1E-10; + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(); + final long[] doublesIn = {10, 20, 30, 40}; + for (int i = 0; i < doublesIn.length; i++) { sketch.update(doublesIn[i]); } + long[] sp = new long[] { 10, 20, 30, 40 }; + println("SplitPoints:"); + for (int i = 0; i < sp.length; i++) { + printf("%10d", sp[i]); + } + println(""); + println("INCLUSIVE:"); + double[] cdf = sketch.getCDF(sp, INCLUSIVE); + double[] pmf = sketch.getPMF(sp, INCLUSIVE); + printf("%10s%10s" + LS, "CDF", "PMF"); + for (int i = 0; i < cdf.length; i++) { + printf("%10.2f%10.2f" + LS, cdf[i], pmf[i]); + assertEquals(cdf[i], cdfI[i], toll); + assertEquals(pmf[i], pmfI[i], toll); + } + println("EXCLUSIVE"); + cdf = sketch.getCDF(sp, EXCLUSIVE); + pmf = sketch.getPMF(sp, EXCLUSIVE); + printf("%10s%10s" + LS, "CDF", "PMF"); + for (int i = 0; i < cdf.length; i++) { + printf("%10.2f%10.2f" + LS, cdf[i], pmf[i]); + assertEquals(cdf[i], cdfE[i], toll); + assertEquals(pmf[i], pmfE[i], toll); + } + } + + @Test + public void checkWrapCase1Doubles() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= 21; i++) { sk.update(i); } + + Memory mem = Memory.wrap(sk.toByteArray()); + KllLongsSketch sk2 = KllLongsSketch.wrap(mem); + + assertTrue(mem.isReadOnly()); + assertTrue(sk2.isReadOnly()); + assertFalse(sk2.isDirect()); + } + + @Test + public void checkWritableWrapCase6And2Doubles() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= 21; i++) { sk.update(i); } + + WritableMemory wmem = WritableMemory.writableWrap(KllHelper.toByteArray(sk, true)); + KllLongsSketch sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr); + + assertFalse(wmem.isReadOnly()); + assertFalse(sk2.isReadOnly()); + assertFalse(sk2.isDirect()); + } + + @Test + public void checkKllSketchCase5Doubles() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= 21; i++) { sk.update(i); } + + WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray()); + KllLongsSketch sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr); + + assertFalse(wmem.isReadOnly()); + assertTrue(sk2.isReadOnly()); + assertFalse(sk2.isDirect()); + } + + @Test + public void checkKllSketchCase3Doubles() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= 21; i++) { sk.update(i); } + + Memory mem = Memory.wrap(KllHelper.toByteArray(sk, true)); + WritableMemory wmem = (WritableMemory) mem; + KllLongsSketch sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr); + + assertTrue(wmem.isReadOnly()); + assertTrue(sk2.isReadOnly()); + assertFalse(sk2.isDirect()); + } + + @Test + public void checkKllSketchCase7Doubles() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= 21; i++) { sk.update(i); } + + Memory mem = Memory.wrap(KllHelper.toByteArray(sk, true)); + WritableMemory wmem = (WritableMemory) mem; + KllLongsSketch sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr); + + assertTrue(wmem.isReadOnly()); + assertTrue(sk2.isReadOnly()); + assertFalse(sk2.isDirect()); + } + + @Test + public void checkReadOnlyExceptions() { + int[] intArr = new int[0]; + int intV = 2; + int idx = 1; + KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(20); + Memory mem = Memory.wrap(sk1.toByteArray()); + KllLongsSketch sk2 = KllLongsSketch.wrap(mem); + try { sk2.setLevelsArray(intArr); fail(); } catch (SketchesArgumentException e) { } + try { sk2.setLevelsArrayAt(idx,intV); fail(); } catch (SketchesArgumentException e) { } + } + + @Test + public void checkIsSameResource() { + int cap = 128; + WritableMemory wmem = WritableMemory.allocate(cap); + WritableMemory reg1 = wmem.writableRegion(0, 64); + WritableMemory reg2 = wmem.writableRegion(64, 64); + assertFalse(reg1 == reg2); + assertFalse(reg1.isSameResource(reg2)); + + WritableMemory reg3 = wmem.writableRegion(0, 64); + assertFalse(reg1 == reg3); + assertTrue(reg1.isSameResource(reg3)); + + byte[] byteArr1 = KllLongsSketch.newHeapInstance(20).toByteArray(); + reg1.putByteArray(0, byteArr1, 0, byteArr1.length); + KllLongsSketch sk1 = KllLongsSketch.wrap(reg1); + + byte[] byteArr2 = KllLongsSketch.newHeapInstance(20).toByteArray(); + reg2.putByteArray(0, byteArr2, 0, byteArr2.length); + assertFalse(sk1.isSameResource(reg2)); + + byte[] byteArr3 = KllLongsSketch.newHeapInstance(20).toByteArray(); + reg3.putByteArray(0, byteArr3, 0, byteArr3.length); + assertTrue(sk1.isSameResource(reg3)); + } + + @Test + public void checkSortedViewAfterReset() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(20); + sk.update(1L); + LongsSortedView sv = sk.getSortedView(); + long dsv = sv.getQuantile(1.0, INCLUSIVE); + assertEquals(dsv, 1L); + sk.reset(); + try { sk.getSortedView(); fail(); } catch (SketchesArgumentException e) { } + } + + @Test + public void checkVectorUpdate() { + boolean withLevels = false; + boolean withLevelsAndItems = true; + int k = 20; + int n = 108; + int maxVsz = 40; //max vector size + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + int j = 1; + int rem; + while ((rem = n - j + 1) > 0) { + int vecSz = min(rem, maxVsz); + long[] v = new long[vecSz]; + for (int i = 0; i < vecSz; i++) { v[i] = j++; } + sk.update(v, 0, vecSz); + } + println(LS + "#<<< END STATE # >>>"); + println(sk.toString(withLevels, withLevelsAndItems)); + println(""); + assertEquals(sk.getN(), 108); + assertEquals(sk.getMaxItem(), 108L); + assertEquals(sk.getMinItem(), 1L); + } + + @Test + public void vectorizedUpdates() { + final int trials = 1; + final int M = 1; //number of vectors + final int N = 1000; //vector size + final int K = 256; + final long[] values = new long[N]; + long vIn = 1L; + long totN = 0; + final long startTime = System.nanoTime(); + for (int t = 0; t < trials; t++) { + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(K); + for (int m = 0; m < M; m++) { + for (int n = 0; n < N; n++) { + values[n] = vIn++; //fill vector + } + sketch.update(values, 0, N); //vector input + } + totN = sketch.getN(); + assertEquals(totN, M * N); + assertEquals(sketch.getMinItem(), 1L); + assertEquals(sketch.getMaxItem(), totN); + assertEquals(sketch.getQuantile(0.5), totN / 2, totN * PMF_EPS_FOR_K_256 * 2.0); //wider tolerance + } + final long runTime = System.nanoTime() - startTime; + println("Vectorized Updates"); + printf(" Vector size : %,12d" + LS, N); + printf(" Num Vectors : %,12d" + LS, M); + printf(" Total Input : %,12d" + LS, totN); + printf(" Run Time mS : %,12.3f" + LS, runTime / 1e6); + final double trialTime = runTime / (1e6 * trials); + printf(" mS / Trial : %,12.3f" + LS, trialTime); + final double updateTime = runTime / (1.0 * totN * trials); + printf(" nS / Update : %,12.3f" + LS, updateTime); + } + + @Test + public void nonVectorizedUpdates() { + final int trials = 1; + final int M = 1; //number of vectors + final int N = 1000; //vector size + final int K = 256; + final long[] values = new long[N]; + long vIn = 1L; + long totN = 0; + final long startTime = System.nanoTime(); + for (int t = 0; t < trials; t++) { + final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(K); + for (int m = 0; m < M; m++) { + for (int n = 0; n < N; n++) { + values[n] = vIn++; //fill vector + } + for (int i = 0; i < N; i++) { + sketch.update(values[i]); //single item input + } + } + totN = sketch.getN(); + assertEquals(totN, M * N); + assertEquals(sketch.getMinItem(), 1L); + assertEquals(sketch.getMaxItem(), totN); + assertEquals(sketch.getQuantile(0.5), totN / 2, totN * PMF_EPS_FOR_K_256 * 2.0); //wider tolerance + } + final long runTime = System.nanoTime() - startTime; + println("Vectorized Updates"); + printf(" Vector size : %,12d" + LS, N); + printf(" Num Vectors : %,12d" + LS, M); + printf(" Total Input : %,12d" + LS, totN); + printf(" Run Time mS : %,12.3f" + LS, runTime / 1e6); + final double trialTime = runTime / (1e6 * trials); + printf(" mS / Trial : %,12.3f" + LS, trialTime); + final double updateTime = runTime / (1.0 * totN * trials); + printf(" nS / Update : %,12.3f" + LS, updateTime); + } + + private final static boolean enablePrinting = false; + + /** + * @param format the format + * @param args the args + */ + private static final void printf(final String format, final Object ...args) { + if (enablePrinting) { System.out.printf(format, args); } + } + + /** + * @param o the Object to println + */ + private static final void println(final Object o) { + if (enablePrinting) { System.out.println(o.toString()); } + } +} diff --git a/src/test/java/org/apache/datasketches/kll/KllMiscDirectLongsTest.java b/src/test/java/org/apache/datasketches/kll/KllMiscDirectLongsTest.java new file mode 100644 index 000000000..366f93f7d --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllMiscDirectLongsTest.java @@ -0,0 +1,459 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; +import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.quantilescommon.LongsSortedView; +import org.apache.datasketches.quantilescommon.LongsSortedViewIterator; +import org.testng.annotations.Test; + +public class KllMiscDirectLongsTest { + static final String LS = System.getProperty("line.separator"); + private static final DefaultMemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); + + @Test + public void checkBounds() { + final KllLongsSketch kll = getDirectLongsSketch(200, 0); + for (int i = 0; i < 1000; i++) { + kll.update(i); + } + final double eps = kll.getNormalizedRankError(false); + final long est = kll.getQuantile(0.5); + final long ub = kll.getQuantileUpperBound(0.5); + final long lb = kll.getQuantileLowerBound(0.5); + assertEquals(ub, kll.getQuantile(.5 + eps)); + assertEquals(lb, kll.getQuantile(0.5 - eps)); + println("Ext : " + est); + println("UB : " + ub); + println("LB : " + lb); + final double rest = kll.getRank(est); + final double restUB = kll.getRankUpperBound(rest); + final double restLB = kll.getRankLowerBound(rest); + assertTrue(restUB - rest < (2 * eps)); + assertTrue(rest - restLB < (2 * eps)); + } + + //@Test //enable static println(..) for visual checking + public void visualCheckToString() { + final int k = 20; + final KllLongsSketch sk = getDirectLongsSketch(k, 0); + for (int i = 0; i < 10; i++) { sk.update(i + 1); } + println(sk.toString(true, true)); + + final KllLongsSketch sk2 = getDirectLongsSketch(k, 0); + for (int i = 0; i < 400; i++) { sk2.update(i + 1); } + println("\n" + sk2.toString(true, true)); + + sk2.merge(sk); + final String s2 = sk2.toString(true, true); + println(LS + s2); + } + + @Test + public void viewDirectCompactions() { + int k = 20; + int u = 108; + KllLongsSketch sk = getDirectLongsSketch(k, 0); + for (int i = 1; i <= u; i++) { + sk.update(i); + if (sk.levelsArr[0] == 0) { + println(sk.toString(true, true)); + sk.update(++i); + println(sk.toString(true, true)); + assertEquals(sk.getLongItemsArray()[sk.levelsArr[0]], i); + } + } + } + + @Test + public void viewCompactionAndSortedView() { + int k = 20; + KllLongsSketch sk = getDirectLongsSketch(k, 0); + show(sk, 20); + LongsSortedView sv = sk.getSortedView(); + LongsSortedViewIterator itr = sv.iterator(); + printf("%12s%12s\n", "Value", "CumWeight"); + while (itr.next()) { + long v = itr.getQuantile(); + long wt = itr.getWeight(); + printf("%12d%12d\n", v, wt); + } + } + + private static void show(final KllLongsSketch sk, int limit) { + int i = (int) sk.getN(); + for ( ; i < limit; i++) { sk.update(i + 1); } + println(sk.toString(true, true)); + } + + @Test + public void checkSketchInitializeLongHeap() { + int k = 20; //don't change this + KllLongsSketch sk; + + //println("#### CASE: LONG FULL HEAP"); + sk = getDirectLongsSketch(k, 0); + for (int i = 1; i <= k + 1; i++) { sk.update(i); } + //println(sk.toString(true, true)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), k + 1); + assertEquals(sk.getNumRetained(), 11); + assertFalse(sk.isEmpty()); + assertTrue(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 33); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3); + assertEquals(sk.getMaxItem(), 21L); + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getNumLevels(), 2); + assertFalse(sk.isLevelZeroSorted()); + + //println("#### CASE: LONG HEAP EMPTY"); + sk = getDirectLongsSketch(k, 0); + //println(sk.toString(true, true)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumRetained(), 0); + assertTrue(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) { } + try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) { } + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + + //println("#### CASE: LONG HEAP SINGLE"); + sk = getDirectLongsSketch(k, 0); + sk.update(1); + //println(sk.toString(true, true)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 1); + assertEquals(sk.getNumRetained(), 1); + assertFalse(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + assertEquals(sk.getMaxItem(), 1L); + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + } + + @Test + public void checkSketchInitializeLongHeapifyCompactMem() { + int k = 20; //don't change this + KllLongsSketch sk; + KllLongsSketch sk2; + byte[] compBytes; + WritableMemory wmem; + + //println("#### CASE: LONG FULL HEAPIFIED FROM COMPACT"); + sk2 = getDirectLongsSketch(k, 0); + for (int i = 1; i <= k + 1; i++) { sk2.update(i); } + //println(sk.toString(true, true)); + compBytes = sk2.toByteArray(); + wmem = WritableMemory.writableWrap(compBytes); + //println(KllPreambleUtil.toString(wmem)); + sk = KllLongsSketch.heapify(wmem); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), k + 1); + assertEquals(sk.getNumRetained(), 11); + assertFalse(sk.isEmpty()); + assertTrue(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 33); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3); + assertEquals(sk.getMaxItem(), 21L); + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getNumLevels(), 2); + assertFalse(sk.isLevelZeroSorted()); + + //println("#### CASE: LONG EMPTY HEAPIFIED FROM COMPACT"); + sk2 = getDirectLongsSketch(k, 0); + //println(sk.toString(true, true)); + compBytes = sk2.toByteArray(); + wmem = WritableMemory.writableWrap(compBytes); + //println(KllPreambleUtil.toString(wmem)); + sk = KllLongsSketch.heapify(wmem); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumRetained(), 0); + assertTrue(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) { } + try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) { } + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + + //println("#### CASE: LONG SINGLE HEAPIFIED FROM COMPACT"); + sk2 = getDirectLongsSketch(k, 0); + sk2.update(1); + //println(sk2.toString(true, true)); + compBytes = sk2.toByteArray(); + wmem = WritableMemory.writableWrap(compBytes); + //println(KllPreambleUtil.toString(wmem)); + sk = KllLongsSketch.heapify(wmem); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 1); + assertEquals(sk.getNumRetained(), 1); + assertFalse(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + assertEquals(sk.getMaxItem(), 1L); + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + } + + @Test + public void checkSketchInitializeLongHeapifyUpdatableMem() { + int k = 20; //don't change this + KllLongsSketch sk; + KllLongsSketch sk2; + byte[] compBytes; + WritableMemory wmem; + + //println("#### CASE: LONG FULL HEAPIFIED FROM UPDATABLE"); + sk2 = getDirectLongsSketch(k, 0); + for (int i = 1; i <= k + 1; i++) { sk2.update(i); } + //println(sk2.toString(true, true)); + compBytes = KllHelper.toByteArray(sk2,true); + wmem = WritableMemory.writableWrap(compBytes); + sk = KllHeapLongsSketch.heapifyImpl(wmem); + //println(sk.toString(true, true)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), k + 1); + assertEquals(sk.getNumRetained(), 11); + assertFalse(sk.isEmpty()); + assertTrue(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 33); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3); + assertEquals(sk.getMaxItem(), 21L); + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getNumLevels(), 2); + assertFalse(sk.isLevelZeroSorted()); + + // println("#### CASE: LONG EMPTY HEAPIFIED FROM UPDATABLE"); + sk2 = getDirectLongsSketch(k, 0); + //println(sk.toString(true, true)); + compBytes = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(compBytes); + //println(KllPreambleUtil.toString(wmem)); + sk = KllHeapLongsSketch.heapifyImpl(wmem); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumRetained(), 0); + assertTrue(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) { } + try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) { } + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + + //println("#### CASE: LONG SINGLE HEAPIFIED FROM UPDATABLE"); + sk2 = getDirectLongsSketch(k, 0); + sk2.update(1); + //println(sk.toString(true, true)); + compBytes = KllHelper.toByteArray(sk2,true); + wmem = WritableMemory.writableWrap(compBytes); + //println(KllPreambleUtil.toString(wmem)); + sk = KllHeapLongsSketch.heapifyImpl(wmem); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 1); + assertEquals(sk.getNumRetained(), 1); + assertFalse(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + assertEquals(sk.getMaxItem(), 1L); + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + } + + @Test + public void checkMemoryToStringLongUpdatable() { + int k = 20; //don't change this + KllLongsSketch sk; + KllLongsSketch sk2; + byte[] upBytes; + byte[] upBytes2; + WritableMemory wmem; + String s; + + println("#### CASE: LONG FULL UPDATABLE"); + sk = getDirectLongsSketch(k, 0); + for (int i = 1; i <= k + 1; i++) { sk.update(i); } + upBytes = KllHelper.toByteArray(sk, true); + wmem = WritableMemory.writableWrap(upBytes); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 1: sketch to byte[]/memory & analyze memory"); + println(s); + sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr); + upBytes2 = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(upBytes2); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above"); + println(s); + assertEquals(upBytes, upBytes2); + + println("#### CASE: LONG EMPTY UPDATABLE"); + sk = getDirectLongsSketch(k, 0); + upBytes = KllHelper.toByteArray(sk, true); + wmem = WritableMemory.writableWrap(upBytes); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 1: sketch to byte[]/memory & analyze memory"); + println(s); + sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr); + upBytes2 = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(upBytes2); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above"); + println(s); + assertEquals(upBytes, upBytes2); + + println("#### CASE: LONG SINGLE UPDATABL"); + sk = getDirectLongsSketch(k, 0); + sk.update(1); + upBytes = KllHelper.toByteArray(sk, true); + wmem = WritableMemory.writableWrap(upBytes); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 1: sketch to byte[]/memory & analyze memory"); + println(s); + sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr); + upBytes2 = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(upBytes2); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above"); + println(s); + assertEquals(upBytes, upBytes2); + } + + @Test + public void checkSimpleMerge() { + int k = 20; + int n1 = 21; + int n2 = 21; + KllLongsSketch sk1 = getDirectLongsSketch(k, 0); + KllLongsSketch sk2 = getDirectLongsSketch(k, 0); + for (int i = 1; i <= n1; i++) { + sk1.update(i); + } + for (int i = 1; i <= n2; i++) { + sk2.update(i + 100); + } + println(sk1.toString(true, true)); + println(sk2.toString(true, true)); + sk1.merge(sk2); + println(sk1.toString(true, true)); + assertEquals(sk1.getMaxItem(), 121L); + assertEquals(sk1.getMinItem(), 1L); + } + + @Test + public void checkSizes() { + KllLongsSketch sk = getDirectLongsSketch(20, 0); + for (int i = 1; i <= 21; i++) { sk.update(i); } + //println(sk.toString(true, true)); + byte[] byteArr1 = KllHelper.toByteArray(sk, true); + int size1 = sk.currentSerializedSizeBytes(true); + assertEquals(size1, byteArr1.length); + byte[] byteArr2 = sk.toByteArray(); + int size2 = sk.currentSerializedSizeBytes(false); + assertEquals(size2, byteArr2.length); + } + + @Test + public void checkNewInstance() { + int k = 200; + WritableMemory dstMem = WritableMemory.allocate(3000); + KllLongsSketch sk = KllLongsSketch.newDirectInstance(k, dstMem, memReqSvr); + for (int i = 1; i <= 10_000; i++) {sk.update(i); } + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getMaxItem(), 10000L); + //println(sk.toString(true, true)); + } + + @Test + public void checkDifferentM() { + int k = 20; + int m = 4; + WritableMemory dstMem = WritableMemory.allocate(1000); + KllLongsSketch sk = KllDirectLongsSketch.newDirectUpdatableInstance(k, m, dstMem, memReqSvr); + for (int i = 1; i <= 200; i++) {sk.update(i); } + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getMaxItem(), 200L); + } + + private static KllLongsSketch getDirectLongsSketch(final int k, final int n) { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= n; i++) { sk.update(i); } + byte[] byteArr = KllHelper.toByteArray(sk, true); + WritableMemory wmem = WritableMemory.writableWrap(byteArr); + KllLongsSketch dfsk = KllLongsSketch.writableWrap(wmem, memReqSvr); + return dfsk; + } + + @Test + public void printlnTest() { + String s = "PRINTING: printf in " + this.getClass().getName(); + println(s); + printf("%s\n", s); + } + + private final static boolean enablePrinting = false; + + /** + * @param format the format + * @param args the args + */ + private static final void printf(final String format, final Object ...args) { + if (enablePrinting) { System.out.printf(format, args); } + } + + /** + * @param o the Object to println + */ + private static final void println(final Object o) { + if (enablePrinting) { System.out.println(o.toString()); } + } + +} diff --git a/src/test/java/org/apache/datasketches/kll/KllMiscLongsTest.java b/src/test/java/org/apache/datasketches/kll/KllMiscLongsTest.java new file mode 100644 index 000000000..018ad91db --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllMiscLongsTest.java @@ -0,0 +1,790 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.kll.KllDirectLongsSketch.KllDirectCompactLongsSketch; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.MemoryRequestServer; +import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.quantilescommon.LongsSortedView; +import org.apache.datasketches.quantilescommon.LongsSortedViewIterator; +import org.testng.annotations.Test; + +import static org.apache.datasketches.common.Util.LS; +import static org.apache.datasketches.common.Util.bitAt; +import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH; +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +/** + * @author Lee Rhodes + */ +public class KllMiscLongsTest { + private final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); + + @Test + public void checkSortedViewConstruction() { + final KllLongsSketch kll = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= 20; i++) { kll.update(i); } + LongsSortedView fsv = kll.getSortedView(); + long[] cumWeights = fsv.getCumulativeWeights(); + long[] values = fsv.getQuantiles(); + assertEquals(cumWeights.length, 20); + assertEquals(values.length, 20); + for (int i = 0; i < 20; i++) { + assertEquals(cumWeights[i], i + 1); + assertEquals(values[i], i + 1); + } + } + + @Test //set static enablePrinting = true for visual checking + public void checkBounds() { + final KllLongsSketch kll = KllLongsSketch.newHeapInstance(); //default k = 200 + for (int i = 0; i < 1000; i++) { + kll.update(i); + } + final double eps = kll.getNormalizedRankError(false); + final long est = kll.getQuantile(0.5); + final long ub = kll.getQuantileUpperBound(0.5); + final long lb = kll.getQuantileLowerBound(0.5); + assertEquals(ub, kll.getQuantile(.5 + eps)); + assertEquals(lb, kll.getQuantile(0.5 - eps)); + println("Ext : " + est); + println("UB : " + ub); + println("LB : " + lb); + final double rest = kll.getRank(est); + final double restUB = kll.getRankUpperBound(rest); + final double restLB = kll.getRankLowerBound(rest); + assertTrue(restUB - rest < (2 * eps)); + assertTrue(rest - restLB < (2 * eps)); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void checkHeapifyExceptions1() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(); + WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray()); + wmem.putByte(6, (byte) 3); //corrupt with odd M + KllLongsSketch.heapify(wmem); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void checkHeapifyExceptions2() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(); + WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray()); + wmem.putByte(0, (byte) 1); //corrupt preamble ints, should be 2 + KllLongsSketch.heapify(wmem); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void checkHeapifyExceptions3() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(); + sk.update(1); + sk.update(2); + WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray()); + wmem.putByte(0, (byte) 1); //corrupt preamble ints, should be 5 + KllLongsSketch.heapify(wmem); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void checkHeapifyExceptions4() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(); + WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray()); + wmem.putByte(1, (byte) 0); //corrupt SerVer, should be 1 or 2 + KllLongsSketch.heapify(wmem); + } + + @Test(expectedExceptions = SketchesArgumentException.class) + public void checkHeapifyExceptions5() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(); + WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray()); + wmem.putByte(2, (byte) 0); //corrupt FamilyID, should be 15 + KllLongsSketch.heapify(wmem); + } + + @Test //set static enablePrinting = true for visual checking + public void checkMisc() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(8); + try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) {} //empty + println(sk.toString(true, true)); + for (int i = 0; i < 20; i++) { sk.update(i); } + println(sk.toString(true, true)); + sk.toByteArray(); + final long[] items = sk.getLongItemsArray(); + assertEquals(items.length, 16); + final int[] levels = sk.getLevelsArray(sk.sketchStructure); + assertEquals(levels.length, 3); + assertEquals(sk.getNumLevels(), 2); + } + + @Test //set static enablePrinting = true for visual checking + public void visualCheckToString() { + final KllLongsSketch sk = KllLongsSketch.newHeapInstance(20); + int n = 21; + for (int i = 1; i <= n; i++) { sk.update(i); } + println(sk.toString(true, true)); + assertEquals(sk.getNumLevels(), 2); + assertEquals(sk.getMinItem(), 1); + assertEquals(sk.getMaxItem(), 21); + assertEquals(sk.getNumRetained(), 11); + + final KllLongsSketch sk2 = KllLongsSketch.newHeapInstance(20); + n = 400; + for (int i = 101; i <= n + 100; i++) { sk2.update(i); } + println(LS + sk2.toString(true, true)); + assertEquals(sk2.getNumLevels(), 5); + assertEquals(sk2.getMinItem(), 101); + assertEquals(sk2.getMaxItem(), 500); + assertEquals(sk2.getNumRetained(), 52); + + sk2.merge(sk); + println(LS + sk2.toString(true, true)); + assertEquals(sk2.getNumLevels(), 5); + assertEquals(sk2.getMinItem(), 1); + assertEquals(sk2.getMaxItem(), 500); + assertEquals(sk2.getNumRetained(), 56); + } + + @Test //set static enablePrinting = true for visual checking + public void viewHeapCompactions() { + int k = 20; + int n = 108; + boolean withLevels = false; + boolean withLevelsAndItems = true; + int compaction = 0; + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= n; i++) { + sk.update(i); + if (sk.levelsArr[0] == 0) { + println(LS + "#<<< BEFORE COMPACTION # " + (++compaction) + " >>>"); + println(sk.toString(withLevels, withLevelsAndItems)); + sk.update(++i); + println(LS + "#<<< AFTER COMPACTION # " + (compaction) + " >>>"); + println(sk.toString(withLevels, withLevelsAndItems)); + assertEquals(sk.getLongItemsArray()[sk.levelsArr[0]], i); + } + } + println(LS + "#<<< END STATE # >>>"); + println(sk.toString(withLevels, withLevelsAndItems)); + println(""); + } + + @Test //set static enablePrinting = true for visual checking + public void viewDirectCompactions() { + int k = 20; + int n = 108; + boolean withLevels = false; + boolean withLevelsAndItems = true; + int compaction = 0; + int sizeBytes = KllSketch.getMaxSerializedSizeBytes(k, n, LONGS_SKETCH, true); + WritableMemory wmem = WritableMemory.allocate(sizeBytes); + KllLongsSketch sk = KllLongsSketch.newDirectInstance(k, wmem, memReqSvr); + for (int i = 1; i <= n; i++) { + sk.update(i); + if (sk.levelsArr[0] == 0) { + println(LS + "#<<< BEFORE COMPACTION # " + (++compaction) + " >>>"); + println(sk.toString(withLevels, withLevelsAndItems)); + sk.update(++i); + println(LS + "#<<< AFTER COMPACTION # " + (compaction) + " >>>"); + println(sk.toString(withLevels, withLevelsAndItems)); + assertEquals(sk.getLongItemsArray()[sk.levelsArr[0]], i); + } + } + println(LS + "#<<< END STATE # >>>"); + println(sk.toString(withLevels, withLevelsAndItems)); + println(""); + } + + @Test //set static enablePrinting = true for visual checking + public void viewCompactionAndSortedView() { + int n = 43; + KllLongsSketch sk = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= n; i++) { sk.update(i); } + println(sk.toString(true, true)); + LongsSortedView sv = sk.getSortedView(); + LongsSortedViewIterator itr = sv.iterator(); + println("### SORTED VIEW"); + printf("%6s %12s %12s" + LS, "Idx", "Value", "Weight"); + int i = 0; + while (itr.next()) { + long v = itr.getQuantile(); + long wt = itr.getWeight(); + printf("%6d %12d %12d" + LS, i, v, wt); + i++; + } + assertEquals(sv.getMinItem(), 1L); + assertEquals(sv.getMaxItem(), n); + } + + @Test //set static enablePrinting = true for visual checking + public void checkWeightedUpdates1() { + int k = 20; + int weight = 127; + long item = 10; + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + println(sk.toString(true, true)); + sk.update(item, weight); + println(sk.toString(true, true)); + assertEquals(sk.getNumRetained(), 7); + assertEquals(sk.getN(), weight); + sk.update(item, weight); + println(sk.toString(true, true)); + assertEquals(sk.getNumRetained(), 14); + assertEquals(sk.getN(), 254); + } + + @Test //set static enablePrinting = true for visual checking + public void checkWeightedUpdates2() { + int k = 20; + int initial = 1000; + int weight = 127; + long item = 10; + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= initial; i++) { sk.update(i + 1000); } + println(sk.toString(true, true)); + sk.update(item, weight); + println(sk.toString(true, true)); + assertEquals(sk.getNumRetained(), 65); + assertEquals(sk.getN(), 1127); + + LongsSortedViewIterator itr = sk.getSortedView().iterator(); + println("### SORTED VIEW"); + printf("%12s %12s %12s" + LS, "Value", "Weight", "NaturalRank"); + long cumWt = 0; + while (itr.next()) { + long v = itr.getQuantile(); + long wt = itr.getWeight(); + long natRank = itr.getNaturalRank(INCLUSIVE); + cumWt += wt; + assertEquals(cumWt, natRank); + printf("%12d %12d %12d" + LS, v, wt, natRank); + } + assertEquals(cumWt, sk.getN()); + } + + @Test //set static enablePrinting = true for visual checking + public void checkCreateItemsArray() { //used with weighted updates + long item = 10; + int weight = 108; + long[] itemsArr = KllLongsHelper.createItemsArray(item, weight); + assertEquals(itemsArr.length, 4); + for (int i = 0; i < itemsArr.length; i++) { itemsArr[i] = item; } + outputItems(itemsArr); + } + + private static void outputItems(long[] itemsArr) { + String[] hdr2 = {"Index", "Value"}; + String hdr2fmt = "%6s %15s" + LS; + String d2fmt = "%6d %15d" + LS; + println("ItemsArr"); + printf(hdr2fmt, (Object[]) hdr2); + for (int i = 0; i < itemsArr.length; i++) { + printf(d2fmt, i, itemsArr[i]); + } + println(""); + } + + @Test //set static enablePrinting = true for visual checking + public void checkCreateLevelsArray() { //used with weighted updates + int weight = 108; + int[] levelsArr = KllHelper.createLevelsArray(weight); + assertEquals(levelsArr.length, 8); + int[] correct = {0,0,0,1,2,2,3,4}; + for (int i = 0; i < levelsArr.length; i++) { + assertEquals(levelsArr[i], correct[i]); + } + outputLevels(weight, levelsArr); + } + + private static void outputLevels(int weight, int[] levelsArr) { + String[] hdr = {"Lvl", "StartAdr", "BitPattern", "Weight"}; + String hdrfmt = "%3s %9s %10s %s" + LS; + String dfmt = "%3d %9d %10d %d" + LS; + String dfmt_2 = "%3d %9d %s" + LS; + println("Count = " + weight + " => " + (Integer.toBinaryString(weight))); + println("LevelsArr"); + printf(hdrfmt, (Object[]) hdr); + for (int i = 0; i < levelsArr.length; i++) { + if (i == levelsArr.length - 1) { printf(dfmt_2, i, levelsArr[i], "ItemsArr.length"); } + else { + int j = bitAt(weight, i); + printf(dfmt, i, levelsArr[i], j, 1 << (i)); + } + } + println(""); + } + + @Test + public void viewMemorySketchData() { + int k = 20; + int n = 109; + boolean withLevels = true; + boolean withLevelsAndItems = true; + KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= n; i++) { sk.update(i); } + byte[] byteArr = sk.toByteArray(); + Memory mem = Memory.wrap(byteArr); + KllLongsSketch fltSk = KllLongsSketch.wrap(mem); + println(fltSk.toString(withLevels, withLevelsAndItems)); + assertEquals(fltSk.getN(), n); + } + + @Test //set static enablePrinting = true for visual checking + public void checkIntCapAux() { + String[] hdr = {"level", "depth", "wt", "cap", "(end)", "MaxN"}; + String hdrFmt = "%6s %6s %28s %10s %10s %34s" + LS; + String dataFmt = "%6d %6d %,28d %,10d %,10d %,34d" + LS; + int k = 1000; + int m = 8; + int numLevels = 20; + println("k=" + k + ", m=" + m + ", numLevels=" + numLevels); + printf(hdrFmt, (Object[]) hdr); + long maxN = 0; + long[] correct = {0,1,1,2,2,3,5,8,12,17,26,39,59,88,132,198,296,444,667,1000}; + for (int i = 0; i < numLevels; i++) { + int depth = numLevels - i - 1; + long cap = KllHelper.intCapAux(k, depth); + long end = Math.max(m, cap); + long wt = 1L << i; + maxN += wt * end; + printf(dataFmt, i, depth, wt, cap, end, maxN); + assertEquals(cap, correct[i]); + } + } + + @Test //set static enablePrinting = true for visual checking + public void checkIntCapAuxAux() { + String[] hdr = {"d","twoK","2k*2^d","3^d","tmp=2k*2^d/3^d","(tmp + 1)/2", "(end)"}; + String hdrFmt = "%6s %10s %20s %20s %15s %12s %10s" + LS; + String dataFmt = "%6d %10d %,20d %,20d %15d %12d %10d" + LS; + long k = (1L << 16) - 1L; + long m = 8; + println("k = " + k + ", m = " + m); + printf(hdrFmt, (Object[]) hdr); + long[] correct = + {65535,43690,29127,19418,12945,8630,5753,3836,2557,1705,1136,758,505,337,224,150,100,67,44,30,20,13,9,6,4,3,2,1,1,1,0}; + for (int i = 0; i < 31; i++) { + long twoK = k << 1; + long twoKxtwoD = twoK << i; + long threeToD = KllHelper.powersOfThree[i]; + long tmp = twoKxtwoD / threeToD; + long result = (tmp + 1L) >>> 1; + long end = Math.max(m, result); //performed later + printf(dataFmt, i, twoK, twoKxtwoD, threeToD, tmp, result, end); + assertEquals(result,correct[i]); + assertEquals(result, KllHelper.intCapAuxAux(k, i)); + } + } + + @Test + public void checkGrowLevels() { + KllLongsSketch sk = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= 21; i++) { sk.update(i); } + assertEquals(sk.getNumLevels(), 2); + assertEquals(sk.getLongItemsArray().length, 33); + assertEquals(sk.getLevelsArray(sk.sketchStructure)[2], 33); + } + + @Test //set static enablePrinting = true for visual checking + public void checkSketchInitializeLongHeap() { + int k = 20; //don't change this + KllLongsSketch sk; + + println("#### CASE: LONG FULL HEAP"); + sk = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= k + 1; i++) { sk.update(i); } + println(sk.toString(true, true)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), k + 1); + assertEquals(sk.getNumRetained(), 11); + assertFalse(sk.isEmpty()); + assertTrue(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 33); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3); + assertEquals(sk.getMaxItem(), 21L); + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getNumLevels(), 2); + assertFalse(sk.isLevelZeroSorted()); + + println("#### CASE: LONG HEAP EMPTY"); + sk = KllLongsSketch.newHeapInstance(k); + println(sk.toString(true, true)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumRetained(), 0); + assertTrue(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) { } + try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) { } + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + + println("#### CASE: LONG HEAP SINGLE"); + sk = KllLongsSketch.newHeapInstance(k); + sk.update(1); + println(sk.toString(true, true)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 1); + assertEquals(sk.getNumRetained(), 1); + assertFalse(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + assertEquals(sk.getMaxItem(), 1L); + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + } + + @Test //set static enablePrinting = true for visual checking + public void checkSketchInitializeLongHeapifyCompactMem() { + int k = 20; //don't change this + KllLongsSketch sk; + KllLongsSketch sk2; + byte[] compBytes; + WritableMemory wmem; + + println("#### CASE: LONG FULL HEAPIFIED FROM COMPACT"); + sk2 = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= k + 1; i++) { sk2.update(i); } + //println(sk2.toString(true, true)); + compBytes = sk2.toByteArray(); + wmem = WritableMemory.writableWrap(compBytes); + println(KllPreambleUtil.toString(wmem, LONGS_SKETCH, true)); + sk = KllLongsSketch.heapify(wmem); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), k + 1); + assertEquals(sk.getNumRetained(), 11); + assertFalse(sk.isEmpty()); + assertTrue(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 33); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3); + assertEquals(sk.getMaxItem(), 21); + assertEquals(sk.getMinItem(), 1); + assertEquals(sk.getNumLevels(), 2); + assertFalse(sk.isLevelZeroSorted()); + + println("#### CASE: LONG EMPTY HEAPIFIED FROM COMPACT"); + sk2 = KllLongsSketch.newHeapInstance(k); + //println(sk.toString(true, true)); + compBytes = sk2.toByteArray(); + wmem = WritableMemory.writableWrap(compBytes); + //println(KllPreambleUtil.toString(wmem, LONGS_SKETCH, true)); + sk = KllLongsSketch.heapify(wmem); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumRetained(), 0); + assertTrue(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) { } + try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) { } + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + + println("#### CASE: LONG SINGLE HEAPIFIED FROM COMPACT"); + sk2 = KllLongsSketch.newHeapInstance(k); + sk2.update(1); + //println(sk2.toString(true, true)); + compBytes = sk2.toByteArray(); + wmem = WritableMemory.writableWrap(compBytes); + //println(KllPreambleUtil.toString(wmem, LONGS_SKETCH, true)); + sk = KllLongsSketch.heapify(wmem); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 1); + assertEquals(sk.getNumRetained(), 1); + assertFalse(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + assertEquals(sk.getMaxItem(), 1L); + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + } + + @Test //set static enablePrinting = true for visual checking + public void checkSketchInitializeLongHeapifyUpdatableMem() { + int k = 20; //don't change this + KllLongsSketch sk; + KllLongsSketch sk2; + byte[] compBytes; + WritableMemory wmem; + + println("#### CASE: LONG FULL HEAPIFIED FROM UPDATABLE"); + sk2 = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= k + 1; i++) { sk2.update(i); } + compBytes = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(compBytes); + println(KllPreambleUtil.toString(wmem, LONGS_SKETCH, true)); + sk = KllHeapLongsSketch.heapifyImpl(wmem); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), k + 1); + assertEquals(sk.getNumRetained(), 11); + assertFalse(sk.isEmpty()); + assertTrue(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 33); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3); + assertEquals(sk.getMaxItem(), 21); + assertEquals(sk.getMinItem(), 1); + assertEquals(sk.getNumLevels(), 2); + assertFalse(sk.isLevelZeroSorted()); + + println("#### CASE: LONG EMPTY HEAPIFIED FROM UPDATABLE"); + sk2 = KllLongsSketch.newHeapInstance(k); + //println(sk.toString(true, true)); + compBytes = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(compBytes); + //println(KllPreambleUtil.toString(wmem, LONGS_SKETCH, true)); + sk = KllHeapLongsSketch.heapifyImpl(wmem); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumRetained(), 0); + assertTrue(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) { } + try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) { } + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + + println("#### CASE: LONG SINGLE HEAPIFIED FROM UPDATABLE"); + sk2 = KllLongsSketch.newHeapInstance(k); + sk2.update(1); + //println(sk.toString(true, true)); + compBytes = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(compBytes); + //println(KllPreambleUtil.toString(wmem, LONGS_SKETCH, true)); + sk = KllHeapLongsSketch.heapifyImpl(wmem); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 1); + assertEquals(sk.getNumRetained(), 1); + assertFalse(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getMinK(), k); + assertEquals(sk.getLongItemsArray().length, 20); + assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2); + assertEquals(sk.getMaxItem(), 1L); + assertEquals(sk.getMinItem(), 1L); + assertEquals(sk.getNumLevels(), 1); + assertFalse(sk.isLevelZeroSorted()); + } + + @Test //set static enablePrinting = true for visual checking + public void checkMemoryToStringLongCompact() { + int k = 20; //don't change this + KllLongsSketch sk; + KllLongsSketch sk2; + byte[] compBytes; + byte[] compBytes2; + WritableMemory wmem; + String s; + + println("#### CASE: LONG FULL COMPACT"); + sk = KllLongsSketch.newHeapInstance(k); + for (int i = 1; i <= k + 1; i++) { sk.update(i); } + compBytes = sk.toByteArray(); + wmem = WritableMemory.writableWrap(compBytes); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 1: sketch to byte[]/memory & analyze memory"); + println(s); + sk2 = KllLongsSketch.heapify(wmem); + compBytes2 = sk2.toByteArray(); + wmem = WritableMemory.writableWrap(compBytes2); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above"); + println(s); + assertEquals(compBytes, compBytes2); + + println("#### CASE: LONG EMPTY COMPACT"); + sk = KllLongsSketch.newHeapInstance(k); + compBytes = sk.toByteArray(); + wmem = WritableMemory.writableWrap(compBytes); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 1: sketch to byte[]/memory & analyze memory"); + println(s); + sk2 = KllLongsSketch.heapify(wmem); + compBytes2 = sk2.toByteArray(); + wmem = WritableMemory.writableWrap(compBytes2); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above"); + println(s); + assertEquals(compBytes, compBytes2); + + println("#### CASE: LONG SINGLE COMPACT"); + sk = KllLongsSketch.newHeapInstance(k); + sk.update(1); + compBytes = sk.toByteArray(); + wmem = WritableMemory.writableWrap(compBytes); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 1: sketch to byte[]/memory & analyze memory"); + println(s); + sk2 = KllLongsSketch.heapify(wmem); + compBytes2 = sk2.toByteArray(); + wmem = WritableMemory.writableWrap(compBytes2); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above"); + println(s); + assertEquals(compBytes, compBytes2); + } + + @Test //set static enablePrinting = true for visual checking + public void checkMemoryToStringLongUpdatable() { + int k = 20; //don't change this + KllLongsSketch sk; + KllLongsSketch sk2; + byte[] upBytes; + byte[] upBytes2; + WritableMemory wmem; + String s; + + println("#### CASE: LONG FULL UPDATABLE"); + sk = KllLongsSketch.newHeapInstance(20); + for (int i = 1; i <= k + 1; i++) { sk.update(i); } + upBytes = KllHelper.toByteArray(sk, true); + wmem = WritableMemory.writableWrap(upBytes); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 1: sketch to byte[]/memory & analyze memory"); + println(s); + sk2 = KllHeapLongsSketch.heapifyImpl(wmem); + upBytes2 = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(upBytes2); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above"); + println(s); //note: heapify does not copy free space, while toUpdatableByteArray does + assertEquals(sk.getN(), sk2.getN()); + assertEquals(sk.getMinItem(), sk2.getMinItem()); + assertEquals(sk.getMaxItem(), sk2.getMaxItem()); + assertEquals(sk.getNumRetained(), sk2.getNumRetained()); + + println("#### CASE: LONG EMPTY UPDATABLE"); + sk = KllLongsSketch.newHeapInstance(k); + upBytes = KllHelper.toByteArray(sk, true); + wmem = WritableMemory.writableWrap(upBytes); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 1: sketch to byte[]/memory & analyze memory"); + println(s); + sk2 = KllHeapLongsSketch.heapifyImpl(wmem); + upBytes2 = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(upBytes2); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above"); + println(s); + assertEquals(upBytes, upBytes2); + + println("#### CASE: LONG SINGLE UPDATABLE"); + sk = KllLongsSketch.newHeapInstance(k); + sk.update(1); + upBytes = KllHelper.toByteArray(sk, true); + wmem = WritableMemory.writableWrap(upBytes); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 1: sketch to byte[]/memory & analyze memory"); + println(s); + sk2 = KllHeapLongsSketch.heapifyImpl(wmem); + upBytes2 = KllHelper.toByteArray(sk2, true); + wmem = WritableMemory.writableWrap(upBytes2); + s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true); + println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above"); + println(s); + assertEquals(upBytes, upBytes2); + } + + @Test + public void checkSimpleMerge() { + int k = 20; + int m = 8; + int n1 = 21; + int n2 = 43; + WritableMemory wmem = WritableMemory.allocate(3000); + WritableMemory wmem2 = WritableMemory.allocate(3000); + + KllLongsSketch sk1 = KllDirectLongsSketch.newDirectUpdatableInstance(k, m, wmem, memReqSvr); + KllLongsSketch sk2 = KllDirectLongsSketch.newDirectUpdatableInstance(k, m, wmem2, memReqSvr); + for (int i = 1; i <= n1; i++) { + sk1.update(i); + } + for (int i = 1; i <= n2; i++) { + sk2.update(i + 100); + } + sk1.merge(sk2); + assertEquals(sk1.getMinItem(), 1L); + assertEquals(sk1.getMaxItem(), 143L); + } + + @Test + public void checkGetSingleItem() { + int k = 20; + KllLongsSketch skHeap = KllLongsSketch.newHeapInstance(k); + skHeap.update(1); + assertTrue(skHeap instanceof KllHeapLongsSketch); + assertEquals(skHeap.getLongSingleItem(), 1L); + + WritableMemory srcMem = WritableMemory.writableWrap(KllHelper.toByteArray(skHeap, true)); + KllLongsSketch skDirect = KllLongsSketch.writableWrap(srcMem, memReqSvr); + assertTrue(skDirect instanceof KllDirectLongsSketch); + assertEquals(skDirect.getLongSingleItem(), 1L); + + Memory srcMem2 = Memory.wrap(skHeap.toByteArray()); + KllLongsSketch skCompact = KllLongsSketch.wrap(srcMem2); + assertTrue(skCompact instanceof KllDirectCompactLongsSketch); + assertEquals(skCompact.getLongSingleItem(), 1L); + } + + @Test + public void printlnTest() { + String s = "PRINTING: printf in " + this.getClass().getName(); + println(s); + printf("%s" + LS, s); + } + + private final static boolean enablePrinting = false; + + /** + * @param format the format + * @param args the args + */ + private static final void printf(final String format, final Object ... args) { + if (enablePrinting) { System.out.printf(format, args); } + } + + /** + * @param o the Object to println + */ + private static final void println(final Object o) { + if (enablePrinting) { System.out.println(o.toString()); } + } + +} From 9c54687ca1ed79a847f3514de3b7529cbc3fd102 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 9 Oct 2024 14:44:47 -0700 Subject: [PATCH 16/88] Add all new kll longs files --- .../kll/KllDirectLongsSketch.java | 391 ++++++++++ .../datasketches/kll/KllHeapLongsSketch.java | 315 +++++++++ .../datasketches/kll/KllLongsHelper.java | 483 +++++++++++++ .../datasketches/kll/KllLongsSketch.java | 669 ++++++++++++++++++ .../kll/KllLongsSketchIterator.java | 40 ++ .../LongsSketchSortedView.java | 137 ++++ .../quantilescommon/LongsSortedView.java | 175 +++++ .../LongsSortedViewIterator.java | 51 ++ .../quantilescommon/QuantilesLongsAPI.java | 300 ++++++++ .../QuantilesLongsSketchIterator.java | 40 ++ 10 files changed, 2601 insertions(+) create mode 100644 src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java create mode 100644 src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java create mode 100644 src/main/java/org/apache/datasketches/kll/KllLongsHelper.java create mode 100644 src/main/java/org/apache/datasketches/kll/KllLongsSketch.java create mode 100644 src/main/java/org/apache/datasketches/kll/KllLongsSketchIterator.java create mode 100644 src/main/java/org/apache/datasketches/quantilescommon/LongsSketchSortedView.java create mode 100644 src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java create mode 100644 src/main/java/org/apache/datasketches/quantilescommon/LongsSortedViewIterator.java create mode 100644 src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java create mode 100644 src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsSketchIterator.java diff --git a/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java new file mode 100644 index 000000000..bf91baabd --- /dev/null +++ b/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import static org.apache.datasketches.common.ByteArrayUtil.copyBytes; +import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR; +import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR_SINGLE_ITEM; +import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryK; +import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryLevelZeroSortedFlag; +import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryM; +import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryMinK; +import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryN; +import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryNumLevels; +import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryFamilyID; +import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryK; +import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryLevelZeroSortedFlag; +import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryM; +import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryMinK; +import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryN; +import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryNumLevels; +import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryPreInts; +import static org.apache.datasketches.kll.KllPreambleUtil.setMemorySerVer; +import static org.apache.datasketches.kll.KllSketch.SketchStructure.COMPACT_EMPTY; +import static org.apache.datasketches.kll.KllSketch.SketchStructure.COMPACT_FULL; +import static org.apache.datasketches.kll.KllSketch.SketchStructure.COMPACT_SINGLE; +import static org.apache.datasketches.kll.KllSketch.SketchStructure.UPDATABLE; +import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH; + +import org.apache.datasketches.common.ByteArrayUtil; +import org.apache.datasketches.common.Family; +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.MemoryRequestServer; +import org.apache.datasketches.memory.WritableMemory; + +/** + * This class implements an off-heap, updatable KllLongsSketch using WritableMemory. + * + *

Please refer to the documentation in the package-info:
+ * {@link org.apache.datasketches.kll}

+ * + * @author Lee Rhodes, Kevin Lang + */ +class KllDirectLongsSketch extends KllLongsSketch { + private WritableMemory wmem; + private MemoryRequestServer memReqSvr; + + /** + * Constructs from Memory or WritableMemory already initialized with a sketch image and validated. + * @param wmem the current WritableMemory + * @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory + * @param memVal the MemoryValadate object + */ + KllDirectLongsSketch( + final SketchStructure sketchStructure, + final WritableMemory wmem, + final MemoryRequestServer memReqSvr, + final KllMemoryValidate memVal) { + super(sketchStructure); + this.wmem = wmem; + this.memReqSvr = memReqSvr; + readOnly = (wmem != null && wmem.isReadOnly()) || sketchStructure != UPDATABLE; + levelsArr = memVal.levelsArr; //always converted to writable form. + } + + /** + * Create a new updatable, direct instance of this sketch. + * @param k parameter that controls size of the sketch and accuracy of estimates + * @param m parameter that controls the minimum level width in items. + * @param dstMem the given destination WritableMemory object for use by the sketch + * @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory + * @return a new instance of this sketch + */ + static KllDirectLongsSketch newDirectUpdatableInstance( + final int k, + final int m, + final WritableMemory dstMem, + final MemoryRequestServer memReqSvr) { + setMemoryPreInts(dstMem, UPDATABLE.getPreInts()); + setMemorySerVer(dstMem, UPDATABLE.getSerVer()); + setMemoryFamilyID(dstMem, Family.KLL.getID()); + setMemoryK(dstMem, k); + setMemoryM(dstMem, m); + setMemoryN(dstMem, 0); + setMemoryMinK(dstMem, k); + setMemoryNumLevels(dstMem, 1); + int offset = DATA_START_ADR; + //new Levels array + dstMem.putIntArray(offset, new int[] {k, k}, 0, 2); + offset += 2 * Integer.BYTES; + //new min/max array + dstMem.putLongArray(offset, new long[] {Long.MAX_VALUE, Long.MIN_VALUE}, 0, 2); + offset += 2 * ITEM_BYTES; + //new empty items array + dstMem.putLongArray(offset, new long[k], 0, k); + + final KllMemoryValidate memVal = new KllMemoryValidate(dstMem, LONGS_SKETCH, null); + final WritableMemory wMem = dstMem; + return new KllDirectLongsSketch(UPDATABLE, wMem, memReqSvr, memVal); + } + + //End of Constructors + + @Override + String getItemAsString(final int index) { + if (isEmpty()) { return "Null"; } + return Long.toString(getLongItemsArray()[index]); + } + + @Override + public int getK() { + return getMemoryK(wmem); + } + + //MinMax Methods + + @Override + public long getMaxItem() { + if (sketchStructure == COMPACT_EMPTY || isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + if (sketchStructure == COMPACT_SINGLE) { return getLongSingleItem(); } + //either compact-full or updatable + final int offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure) + ITEM_BYTES; + return wmem.getLong(offset); + } + + @Override + long getMaxItemInternal() { + if (sketchStructure == COMPACT_EMPTY || isEmpty()) { return Long.MAX_VALUE; } + if (sketchStructure == COMPACT_SINGLE) { return getLongSingleItem(); } + //either compact-full or updatable + final int offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure) + ITEM_BYTES; + return wmem.getLong(offset); + } + + @Override + String getMaxItemAsString() { + final long maxItem = getMaxItemInternal(); + return Long.toString(maxItem); + } + + @Override + public long getMinItem() { + if (sketchStructure == COMPACT_EMPTY || isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + if (sketchStructure == COMPACT_SINGLE) { return getLongSingleItem(); } + //either compact-full or updatable + final int offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure); + return wmem.getLong(offset); + } + + @Override + long getMinItemInternal() { + if (sketchStructure == COMPACT_EMPTY || isEmpty()) { return Long.MAX_VALUE; } + if (sketchStructure == COMPACT_SINGLE) { return getLongSingleItem(); } + //either compact-full or updatable + final int offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure); + return wmem.getLong(offset); + } + + @Override + String getMinItemAsString() { + final long minItem = getMinItemInternal(); + return Long.toString(minItem); + } + + @Override + void setMaxItem(final long item) { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + final int offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure) + ITEM_BYTES; + wmem.putLong(offset, item); + } + + @Override + void setMinItem(final long item) { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + final int offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure); + wmem.putLong(offset, item); + } + + //END MinMax Methods + + @Override + public long getN() { + if (sketchStructure == COMPACT_EMPTY) { return 0; } + else if (sketchStructure == COMPACT_SINGLE) { return 1; } + else { return getMemoryN(wmem); } + } + + //other restricted + + @Override //returns updatable, expanded array including free space at bottom + long[] getLongItemsArray() { + final int k = getK(); + if (sketchStructure == COMPACT_EMPTY) { return new long[k]; } + if (sketchStructure == COMPACT_SINGLE) { + final long[] itemsArr = new long[k]; + itemsArr[k - 1] = getLongSingleItem(); + return itemsArr; + } + final int capacityItems = KllHelper.computeTotalItemCapacity(k, getM(), getNumLevels()); + final long[] longItemsArr = new long[capacityItems]; + final int offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure) + 2 * ITEM_BYTES; + final int shift = (sketchStructure == COMPACT_FULL) ? levelsArr[0] : 0; + final int numItems = (sketchStructure == COMPACT_FULL) ? getNumRetained() : capacityItems; + wmem.getLongArray(offset, longItemsArr, shift, numItems); + return longItemsArr; + } + + @Override //returns compact items array of retained items, no free space. + long[] getLongRetainedItemsArray() { + if (sketchStructure == COMPACT_EMPTY) { return new long[0]; } + if (sketchStructure == COMPACT_SINGLE) { return new long[] { getLongSingleItem() }; } + final int numRetained = getNumRetained(); + final long[] longItemsArr = new long[numRetained]; + final int offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure) + 2 * ITEM_BYTES + + (sketchStructure == COMPACT_FULL ? 0 : levelsArr[0] * ITEM_BYTES); + wmem.getLongArray(offset, longItemsArr, 0, numRetained); + return longItemsArr; + } + + @Override + long getLongSingleItem() { + if (!isSingleItem()) { throw new SketchesArgumentException(NOT_SINGLE_ITEM_MSG); } + if (sketchStructure == COMPACT_SINGLE) { + return wmem.getLong(DATA_START_ADR_SINGLE_ITEM); + } + final int offset; + if (sketchStructure == COMPACT_FULL) { + offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure) + 2 * ITEM_BYTES; + } else { //sketchStructure == UPDATABLE + offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure) + (2 + getK() - 1) * ITEM_BYTES; + } + return wmem.getLong(offset); + } + + @Override + int getM() { + return getMemoryM(wmem); + } + + @Override + MemoryRequestServer getMemoryRequestServer() { return memReqSvr; } + + @Override + int getMinK() { + if (sketchStructure == COMPACT_FULL || sketchStructure == UPDATABLE) { return getMemoryMinK(wmem); } + return getK(); + } + + @Override + byte[] getMinMaxByteArr() { + final byte[] bytesOut = new byte[2 * ITEM_BYTES]; + if (sketchStructure == COMPACT_EMPTY) { + ByteArrayUtil.putLongLE(bytesOut, 0, Long.MAX_VALUE); + ByteArrayUtil.putLongLE(bytesOut, ITEM_BYTES, Long.MIN_VALUE); + return bytesOut; + } + final int offset; + if (sketchStructure == COMPACT_SINGLE) { + offset = DATA_START_ADR_SINGLE_ITEM; + wmem.getByteArray(offset, bytesOut, 0, ITEM_BYTES); + copyBytes(bytesOut, 0, bytesOut, ITEM_BYTES, ITEM_BYTES); + return bytesOut; + } + //sketchStructure == UPDATABLE OR COMPACT_FULL + offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure); + wmem.getByteArray(offset, bytesOut, 0, ITEM_BYTES); + wmem.getByteArray(offset + ITEM_BYTES, bytesOut, ITEM_BYTES, ITEM_BYTES); + return bytesOut; + } + + @Override + byte[] getRetainedItemsByteArr() { + if (sketchStructure == COMPACT_EMPTY) { return new byte[0]; } + final long[] lngArr = getLongRetainedItemsArray(); + final byte[] lngByteArr = new byte[lngArr.length * ITEM_BYTES]; + final WritableMemory wmem2 = WritableMemory.writableWrap(lngByteArr); + wmem2.putLongArray(0, lngArr, 0, lngArr.length); + return lngByteArr; + } + + @Override + byte[] getTotalItemsByteArr() { + final long[] lngArr = getLongItemsArray(); + final byte[] lngByteArr = new byte[lngArr.length * ITEM_BYTES]; + final WritableMemory wmem2 = WritableMemory.writableWrap(lngByteArr); + wmem2.putLongArray(0, lngArr, 0, lngArr.length); + return lngByteArr; + } + + @Override + WritableMemory getWritableMemory() { + return wmem; + } + + @Override + void incN(final int increment) { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + setMemoryN(wmem, getMemoryN(wmem) + increment); + } + + @Override + void incNumLevels() { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + int numLevels = getMemoryNumLevels(wmem); + setMemoryNumLevels(wmem, ++numLevels); + } + + @Override + boolean isLevelZeroSorted() { + return getMemoryLevelZeroSortedFlag(wmem); + } + + @Override + void setLongItemsArray(final long[] longItems) { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + final int offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure) + 2 * ITEM_BYTES; + wmem.putLongArray(offset, longItems, 0, longItems.length); + } + + @Override + void setLongItemsArrayAt(final int index, final long item) { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + final int offset = + DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure) + (index + 2) * ITEM_BYTES; + wmem.putLong(offset, item); + } + + @Override + void setLongItemsArrayAt(final int index, final long[] items, final int srcOffset, final int length) { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + final int offset = DATA_START_ADR + getLevelsArrSizeBytes(sketchStructure) + (index + 2) * ITEM_BYTES; + wmem.putLongArray(offset, items, srcOffset, length); + } + + @Override + void setLevelZeroSorted(final boolean sorted) { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + setMemoryLevelZeroSortedFlag(wmem, sorted); + } + + @Override + void setMinK(final int minK) { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + setMemoryMinK(wmem, minK); + } + + @Override + void setN(final long n) { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + setMemoryN(wmem, n); + } + + @Override + void setNumLevels(final int numLevels) { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + setMemoryNumLevels(wmem, numLevels); + } + + @Override + void setWritableMemory(final WritableMemory wmem) { + this.wmem = wmem; + } + + final static class KllDirectCompactLongsSketch extends KllDirectLongsSketch { + + KllDirectCompactLongsSketch( + final SketchStructure sketchStructure, + final Memory srcMem, + final KllMemoryValidate memVal) { + super(sketchStructure, (WritableMemory) srcMem, null, memVal); + } + } + +} diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java new file mode 100644 index 000000000..c4e091342 --- /dev/null +++ b/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java @@ -0,0 +1,315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import static org.apache.datasketches.common.ByteArrayUtil.putLongLE; +import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR; +import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR_SINGLE_ITEM; +import static org.apache.datasketches.kll.KllSketch.SketchStructure.COMPACT_EMPTY; +import static org.apache.datasketches.kll.KllSketch.SketchStructure.COMPACT_FULL; +import static org.apache.datasketches.kll.KllSketch.SketchStructure.COMPACT_SINGLE; +import static org.apache.datasketches.kll.KllSketch.SketchStructure.UPDATABLE; +import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH; + +import java.util.Arrays; +import java.util.Objects; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.MemoryRequestServer; +import org.apache.datasketches.memory.WritableMemory; + +/** + * This class implements an on-heap longs KllSketch. + * + *

Please refer to the documentation in the package-info:
+ * {@link org.apache.datasketches.kll}

+ * + * @author Lee Rhodes, Kevin Lang + */ +final class KllHeapLongsSketch extends KllLongsSketch { + private final int k; // configured size of K. + private final int m; // configured size of M. + private long n; // number of items input into this sketch. + private int minK; // dynamic minK for error estimation after merging with different k. + private boolean isLevelZeroSorted; + private long minLongItem; + private long maxLongItem; + private long[] longItems; + + /** + * New instance heap constructor with a given parameters k and m. + * + * @param k parameter that controls size of the sketch and accuracy of estimates. + * k can be between m and 65535, inclusive. + * @param m parameter controls the minimum level width in items. It can be 2, 4, 6 or 8. + * The DEFAULT_M, which is 8 is recommended. Other sizes of m should be considered + * experimental as they have not been as well characterized. + */ + KllHeapLongsSketch(final int k, final int m) { + super(UPDATABLE); + KllHelper.checkM(m); + KllHelper.checkK(k, m); + this.levelsArr = new int[] {k, k}; + this.readOnly = false; + this.k = k; + this.m = m; + this.n = 0; + this.minK = k; + this.isLevelZeroSorted = false; + this.minLongItem = Long.MAX_VALUE; + this.maxLongItem = Long.MIN_VALUE; + this.longItems = new long[k]; + } + + /** + * Used for creating a temporary sketch for use with weighted updates. + */ + KllHeapLongsSketch(final int k, final int m, final long item, final long weight) { + super(UPDATABLE); + KllHelper.checkM(m); + KllHelper.checkK(k, m); + this.levelsArr = KllHelper.createLevelsArray(weight); + this.readOnly = false; + this.k = k; + this.m = m; + this.n = weight; + this.minK = k; + this.isLevelZeroSorted = false; + this.minLongItem = item; + this.maxLongItem = item; + this.longItems = KllLongsHelper.createItemsArray(item, weight); + } + + /** + * Heapify constructor. + * @param srcMem Memory object that contains data serialized by this sketch. + * @param memValidate the MemoryValidate object + */ + private KllHeapLongsSketch( + final Memory srcMem, + final KllMemoryValidate memValidate) { + super(UPDATABLE); + final SketchStructure memStructure = memValidate.sketchStructure; + this.k = memValidate.k; + this.m = memValidate.m; + this.n = memValidate.n; + this.minK = memValidate.minK; + this.levelsArr = memValidate.levelsArr; //normalized to full + this.isLevelZeroSorted = memValidate.level0SortedFlag; + + if (memStructure == COMPACT_EMPTY) { + minLongItem = Long.MAX_VALUE; + maxLongItem = Long.MIN_VALUE; + longItems = new long[k]; + } + else if (memStructure == COMPACT_SINGLE) { + final long item = srcMem.getLong(DATA_START_ADR_SINGLE_ITEM); + minLongItem = maxLongItem = item; + longItems = new long[k]; + longItems[k - 1] = item; + } + else if (memStructure == COMPACT_FULL) { + int offsetBytes = DATA_START_ADR; + offsetBytes += (levelsArr.length - 1) * Integer.BYTES; //shortened levelsArr + minLongItem = srcMem.getLong(offsetBytes); + offsetBytes += Long.BYTES; + maxLongItem = srcMem.getLong(offsetBytes); + offsetBytes += Long.BYTES; + final int capacityItems = levelsArr[getNumLevels()]; + final int freeSpace = levelsArr[0]; + final int retainedItems = capacityItems - freeSpace; + longItems = new long[capacityItems]; + srcMem.getLongArray(offsetBytes, longItems, freeSpace, retainedItems); + } + else { //(memStructure == UPDATABLE) + int offsetBytes = DATA_START_ADR; + offsetBytes += levelsArr.length * Integer.BYTES; //full levelsArr + minLongItem = srcMem.getLong(offsetBytes); + offsetBytes += Long.BYTES; + maxLongItem = srcMem.getLong(offsetBytes); + offsetBytes += Long.BYTES; + final int capacityItems = levelsArr[getNumLevels()]; + longItems = new long[capacityItems]; + srcMem.getLongArray(offsetBytes, longItems, 0, capacityItems); + } + } + + static KllHeapLongsSketch heapifyImpl(final Memory srcMem) { + Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null"); + final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, LONGS_SKETCH); + return new KllHeapLongsSketch(srcMem, memVal); + } + + //End of constructors + + @Override + String getItemAsString(final int index) { + if (isEmpty()) { return "Null"; } + return Long.toString(longItems[index]); + } + + @Override + public int getK() { return k; } + + //MinMax Methods + + @Override + long getMaxItemInternal() { return maxLongItem; } + + @Override + public long getMaxItem() { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + return maxLongItem; + } + + @Override + String getMaxItemAsString() { + return Long.toString(maxLongItem); + } + + @Override + long getMinItemInternal() { return minLongItem; } + + @Override + public long getMinItem() { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + return minLongItem; + } + + @Override + String getMinItemAsString() { + return Long.toString(minLongItem); + } + + @Override + byte[] getMinMaxByteArr() { + final byte[] bytesOut = new byte[2 * Long.BYTES]; + putLongLE(bytesOut, 0, minLongItem); + putLongLE(bytesOut, Long.BYTES, maxLongItem); + return bytesOut; + } + + @Override + void setMaxItem(final long item) { this.maxLongItem = item; } + + @Override + void setMinItem(final long item) { this.minLongItem = item; } + + //END MinMax Methods + + @Override + public long getN() { return n; } + + //other restricted + + @Override + long[] getLongItemsArray() { return longItems; } + + @Override + long getLongSingleItem() { + if (n != 1L) { throw new SketchesArgumentException(NOT_SINGLE_ITEM_MSG); } + return longItems[k - 1]; + } + + @Override + int getM() { return m; } + + @Override + MemoryRequestServer getMemoryRequestServer() { return null; } + + @Override + int getMinK() { return minK; } + + @Override + byte[] getRetainedItemsByteArr() { + if (isEmpty()) { return new byte[0]; } + final byte[] bytesOut; + if (isSingleItem()) { + bytesOut = new byte[Long.BYTES]; + putLongLE(bytesOut, 0, getLongSingleItem()); + return bytesOut; + } + final int retained = getNumRetained(); + final int bytes = retained * Long.BYTES; + bytesOut = new byte[bytes]; + final WritableMemory wmem = WritableMemory.writableWrap(bytesOut); + wmem.putLongArray(0, longItems, levelsArr[0], retained); + return bytesOut; + } + + @Override + byte[] getTotalItemsByteArr() { + final byte[] byteArr = new byte[longItems.length * Long.BYTES]; + final WritableMemory wmem = WritableMemory.writableWrap(byteArr); + wmem.putLongArray(0, longItems, 0, longItems.length); + return byteArr; + } + + @Override + WritableMemory getWritableMemory() { + return null; + } + + @Override + void incN(final int increment) { n += increment; } + + @Override + void incNumLevels() { + //the heap sketch computes num levels from the array itself, so this is not used on-heap + } + + @Override + boolean isLevelZeroSorted() { return this.isLevelZeroSorted; } + + @Override + void setLongItemsArray(final long[] longItems) { this.longItems = longItems; } + + @Override + void setLongItemsArrayAt(final int index, final long item) { this.longItems[index] = item; } + + @Override + void setLongItemsArrayAt(final int dstIndex, final long[] srcItems, final int srcOffset, final int length) { + System.arraycopy(srcItems, srcOffset, longItems, dstIndex, length); + } + + @Override + void setLevelZeroSorted(final boolean sorted) { this.isLevelZeroSorted = sorted; } + + @Override + void setMinK(final int minK) { this.minK = minK; } + + @Override + void setN(final long n) { this.n = n; } + + @Override + void setNumLevels(final int numLevels) { + //the heap sketch computes num levels from the array itself, so this is not used on-heap + } + + @Override + long[] getLongRetainedItemsArray() { + return Arrays.copyOfRange(longItems, levelsArr[0], levelsArr[getNumLevels()]); + } + + @Override + void setWritableMemory(final WritableMemory wmem) { } + +} diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java b/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java new file mode 100644 index 000000000..ec67b55d4 --- /dev/null +++ b/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java @@ -0,0 +1,483 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import static java.lang.Math.max; +import static java.lang.Math.min; +import static org.apache.datasketches.common.Util.isEven; +import static org.apache.datasketches.common.Util.isOdd; +import static org.apache.datasketches.kll.KllHelper.findLevelToCompact; + +import java.util.Arrays; +import java.util.Random; + +import org.apache.datasketches.memory.WritableMemory; + +/** + * Static methods to support KllLongsSketch + * @author Lee Rhodes + * @author Zac Blanco + */ +final class KllLongsHelper { + + /** + * Create Items Array from given item and weight. + * Used with weighted update only. + * @param item the given item + * @param weight the given weight + * @return the Items Array. + */ + static long[] createItemsArray(final long item, final long weight) { + final int itemsArrLen = Long.bitCount(weight); + final long[] itemsArr = new long[itemsArrLen]; + Arrays.fill(itemsArr, item); + return itemsArr; + } + + /** + * The following code is only valid in the special case of exactly reaching capacity while updating. + * It cannot be used while merging, while reducing k, or anything else. + * @param lngSk the current KllLongsSketch + */ + static void compressWhileUpdatingSketch(final KllLongsSketch lngSk) { + final int level = + findLevelToCompact(lngSk.getK(), lngSk.getM(), lngSk.getNumLevels(), lngSk.levelsArr); + if (level == lngSk.getNumLevels() - 1) { + //The level to compact is the top level, thus we need to add a level. + //Be aware that this operation grows the items array, + //shifts the items data and the level boundaries of the data, + //and grows the levels array and increments numLevels_. + KllHelper.addEmptyTopLevelToCompletelyFullSketch(lngSk); + } + //after this point, the levelsArray will not be expanded, only modified. + final int[] myLevelsArr = lngSk.levelsArr; + final int rawBeg = myLevelsArr[level]; + final int rawEnd = myLevelsArr[level + 1]; + // +2 is OK because we already added a new top level if necessary + final int popAbove = myLevelsArr[level + 2] - rawEnd; + final int rawPop = rawEnd - rawBeg; + final boolean oddPop = isOdd(rawPop); + final int adjBeg = oddPop ? rawBeg + 1 : rawBeg; + final int adjPop = oddPop ? rawPop - 1 : rawPop; + final int halfAdjPop = adjPop / 2; + + //the following is specific to longs + final long[] myLongItemsArray = lngSk.getLongItemsArray(); + if (level == 0) { // level zero might not be sorted, so we must sort it if we wish to compact it + Arrays.sort(myLongItemsArray, adjBeg, adjBeg + adjPop); + } + if (popAbove == 0) { + KllLongsHelper.randomlyHalveUpLongs(myLongItemsArray, adjBeg, adjPop, KllSketch.random); + } else { + KllLongsHelper.randomlyHalveDownLongs(myLongItemsArray, adjBeg, adjPop, KllSketch.random); + KllLongsHelper.mergeSortedLongArrays( + myLongItemsArray, adjBeg, halfAdjPop, + myLongItemsArray, rawEnd, popAbove, + myLongItemsArray, adjBeg + halfAdjPop); + } + + int newIndex = myLevelsArr[level + 1] - halfAdjPop; // adjust boundaries of the level above + lngSk.setLevelsArrayAt(level + 1, newIndex); + + if (oddPop) { + lngSk.setLevelsArrayAt(level, myLevelsArr[level + 1] - 1); // the current level now contains one item + myLongItemsArray[myLevelsArr[level]] = myLongItemsArray[rawBeg]; // namely this leftover guy + } else { + lngSk.setLevelsArrayAt(level, myLevelsArr[level + 1]); // the current level is now empty + } + + // verify that we freed up halfAdjPop array slots just below the current level + assert myLevelsArr[level] == rawBeg + halfAdjPop; + + // finally, we need to shift up the data in the levels below + // so that the freed-up space can be used by level zero + if (level > 0) { + final int amount = rawBeg - myLevelsArr[0]; + System.arraycopy(myLongItemsArray, myLevelsArr[0], myLongItemsArray, myLevelsArr[0] + halfAdjPop, amount); + } + for (int lvl = 0; lvl < level; lvl++) { + newIndex = myLevelsArr[lvl] + halfAdjPop; //adjust boundary + lngSk.setLevelsArrayAt(lvl, newIndex); + } + lngSk.setLongItemsArray(myLongItemsArray); + } + + //assumes readOnly = false and UPDATABLE, called from KllLongsSketch::merge + static void mergeLongsImpl(final KllLongsSketch mySketch, final KllLongsSketch otherLngSk) { + if (otherLngSk.isEmpty()) { return; } + + //capture my key mutable fields before doing any merging + final boolean myEmpty = mySketch.isEmpty(); + final long myMin = mySketch.getMinItemInternal(); + final long myMax = mySketch.getMaxItemInternal(); + final int myMinK = mySketch.getMinK(); + final long finalN = Math.addExact(mySketch.getN(), otherLngSk.getN()); + + //buffers that are referenced multiple times + final int otherNumLevels = otherLngSk.getNumLevels(); + final int[] otherLevelsArr = otherLngSk.levelsArr; + final long[] otherLongItemsArray; + + //MERGE: update this sketch with level0 items from the other sketch + if (otherLngSk.isCompactSingleItem()) { + KllLongsSketch.updateLong(mySketch, otherLngSk.getLongSingleItem()); + otherLongItemsArray = new long[0]; + } else { + otherLongItemsArray = otherLngSk.getLongItemsArray(); + for (int i = otherLevelsArr[0]; i < otherLevelsArr[1]; i++) { + KllLongsSketch.updateLong(mySketch, otherLongItemsArray[i]); + } + } + + //After the level 0 update, we capture the intermediate state of my levels and items arrays... + final int myCurNumLevels = mySketch.getNumLevels(); + final int[] myCurLevelsArr = mySketch.levelsArr; + final long[] myCurLongItemsArray = mySketch.getLongItemsArray(); + + // create aliases in case there are no higher levels + int myNewNumLevels = myCurNumLevels; + int[] myNewLevelsArr = myCurLevelsArr; + long[] myNewLongItemsArray = myCurLongItemsArray; + + //merge higher levels if they exist + if (otherNumLevels > 1 && !otherLngSk.isCompactSingleItem()) { + final int tmpSpaceNeeded = mySketch.getNumRetained() + + KllHelper.getNumRetainedAboveLevelZero(otherNumLevels, otherLevelsArr); + final long[] workbuf = new long[tmpSpaceNeeded]; + + final int provisionalNumLevels = max(myCurNumLevels, otherNumLevels); + + final int ub = max(KllHelper.ubOnNumLevels(finalN), provisionalNumLevels); + final int[] worklevels = new int[ub + 2]; // ub+1 does not work + final int[] outlevels = new int[ub + 2]; + + populateLongWorkArrays(workbuf, worklevels, provisionalNumLevels, + myCurNumLevels, myCurLevelsArr, myCurLongItemsArray, + otherNumLevels, otherLevelsArr, otherLongItemsArray); + + // notice that workbuf is being used as both the input and output + final int[] result = generalLongsCompress(mySketch.getK(), mySketch.getM(), provisionalNumLevels, + workbuf, worklevels, workbuf, outlevels, mySketch.isLevelZeroSorted(), KllSketch.random); + final int targetItemCount = result[1]; //was finalCapacity. Max size given k, m, numLevels + final int curItemCount = result[2]; //was finalPop + + // now we need to finalize the results for mySketch + + //THE NEW NUM LEVELS + myNewNumLevels = result[0]; + assert myNewNumLevels <= ub; // ub may be much bigger + + // THE NEW ITEMS ARRAY + myNewLongItemsArray = (targetItemCount == myCurLongItemsArray.length) + ? myCurLongItemsArray + : new long[targetItemCount]; + final int freeSpaceAtBottom = targetItemCount - curItemCount; + + //shift the new items array create space at bottom + System.arraycopy(workbuf, outlevels[0], myNewLongItemsArray, freeSpaceAtBottom, curItemCount); + final int theShift = freeSpaceAtBottom - outlevels[0]; + + //calculate the new levels array length + final int finalLevelsArrLen; + if (myCurLevelsArr.length < myNewNumLevels + 1) { finalLevelsArrLen = myNewNumLevels + 1; } + else { finalLevelsArrLen = myCurLevelsArr.length; } + + //THE NEW LEVELS ARRAY + myNewLevelsArr = new int[finalLevelsArrLen]; + for (int lvl = 0; lvl < myNewNumLevels + 1; lvl++) { // includes the "extra" index + myNewLevelsArr[lvl] = outlevels[lvl] + theShift; + } + + //MEMORY SPACE MANAGEMENT + if (mySketch.getWritableMemory() != null) { + final WritableMemory wmem = + KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewLongItemsArray.length); + mySketch.setWritableMemory(wmem); + } + } //end of updating levels above level 0 + + //Update Preamble: + mySketch.setN(finalN); + if (otherLngSk.isEstimationMode()) { //otherwise the merge brings over exact items. + mySketch.setMinK(min(myMinK, otherLngSk.getMinK())); + } + + //Update numLevels, levelsArray, items + mySketch.setNumLevels(myNewNumLevels); + mySketch.setLevelsArray(myNewLevelsArr); + mySketch.setLongItemsArray(myNewLongItemsArray); + + //Update min, max items + final long otherMin = otherLngSk.getMinItemInternal(); + final long otherMax = otherLngSk.getMaxItemInternal(); + if (myEmpty) { + mySketch.setMinItem(otherMin); + mySketch.setMaxItem(otherMax); + } else { + mySketch.setMinItem(min(myMin, otherMin)); + mySketch.setMaxItem(max(myMax, otherMax)); + } + assert KllHelper.sumTheSampleWeights(mySketch.getNumLevels(), mySketch.levelsArr) == mySketch.getN(); + } + + private static void mergeSortedLongArrays( //only bufC is modified + final long[] bufA, final int startA, final int lenA, + final long[] bufB, final int startB, final int lenB, + final long[] bufC, final int startC) { + final int lenC = lenA + lenB; + final int limA = startA + lenA; + final int limB = startB + lenB; + final int limC = startC + lenC; + + int a = startA; + int b = startB; + + for (int c = startC; c < limC; c++) { + if (a == limA) { + bufC[c] = bufB[b]; + b++; + } else if (b == limB) { + bufC[c] = bufA[a]; + a++; + } else if (bufA[a] < bufB[b]) { + bufC[c] = bufA[a]; + a++; + } else { + bufC[c] = bufB[b]; + b++; + } + } + assert a == limA; + assert b == limB; + } + + /** + * Validation Method. This must be modified to use the validation test + * @param buf the items array + * @param start data start + * @param length items array length + * @param random instance of Random + */ + //NOTE For validation Method: Need to modify to run. + private static void randomlyHalveDownLongs(final long[] buf, final int start, final int length, + final Random random) { + assert isEven(length); + final int half_length = length / 2; + final int offset = random.nextInt(2); // disable for validation + //final int offset = deterministicOffset(); // enable for validation + int j = start + offset; + for (int i = start; i < (start + half_length); i++) { + buf[i] = buf[j]; + j += 2; + } + } + + /** + * Validation Method. This must be modified to use the validation test + * @param buf the items array + * @param start data start + * @param length items array length + * @param random instance of Random + */ + //NOTE For validation Method: Need to modify to run. + private static void randomlyHalveUpLongs(final long[] buf, final int start, final int length, + final Random random) { + assert isEven(length); + final int half_length = length / 2; + final int offset = random.nextInt(2); // disable for validation + //final int offset = deterministicOffset(); // enable for validation + int j = (start + length) - 1 - offset; + for (int i = (start + length) - 1; i >= (start + half_length); i--) { + buf[i] = buf[j]; + j -= 2; + } + } + + /** + * Compression algorithm used to merge higher levels. + *

Here is what we do for each level:

+ *
  • If it does not need to be compacted, then simply copy it over.
    • + *
  • Otherwise, it does need to be compacted, so... + *
    • Copy zero or one guy over.
      • + *
    • If the level above is empty, halve up.
      • + *
    • Else the level above is nonempty, so halve down, then merge up.
      • + *
    • + *
  • Adjust the boundaries of the level above.
    • + *
+ * + *

It can be proved that generalCompress returns a sketch that satisfies the space constraints + * no matter how much data is passed in. + * We are pretty sure that it works correctly when inBuf and outBuf are the same. + * All levels except for level zero must be sorted before calling this, and will still be + * sorted afterwards. + * Level zero is not required to be sorted before, and may not be sorted afterwards.

+ * + *

This trashes inBuf and inLevels and modifies outBuf and outLevels.

+ * + * @param k The sketch parameter k + * @param m The minimum level size + * @param numLevelsIn provisional number of number of levels = max(this.numLevels, other.numLevels) + * @param inBuf work buffer of size = this.getNumRetained() + other.getNumRetainedAboveLevelZero(). + * This contains the long[] of the other sketch + * @param inLevels work levels array size = ubOnNumLevels(this.n + other.n) + 2 + * @param outBuf the same array as inBuf + * @param outLevels the same size as inLevels + * @param isLevelZeroSorted true if this.level 0 is sorted + * @param random instance of java.util.Random + * @return int array of: {numLevels, targetItemCount, currentItemCount) + */ + // + private static int[] generalLongsCompress( + final int k, + final int m, + final int numLevelsIn, + final long[] inBuf, + final int[] inLevels, + final long[] outBuf, + final int[] outLevels, + final boolean isLevelZeroSorted, + final Random random) { + assert numLevelsIn > 0; // things are too weird if zero levels are allowed + int numLevels = numLevelsIn; + int currentItemCount = inLevels[numLevels] - inLevels[0]; // decreases with each compaction + int targetItemCount = KllHelper.computeTotalItemCapacity(k, m, numLevels); // increases if we add levels + boolean doneYet = false; + outLevels[0] = 0; + int curLevel = -1; + while (!doneYet) { + curLevel++; // start out at level 0 + + // If we are at the current top level, add an empty level above it for convenience, + // but do not actually increment numLevels until later + if (curLevel == (numLevels - 1)) { + inLevels[curLevel + 2] = inLevels[curLevel + 1]; + } + + final int rawBeg = inLevels[curLevel]; + final int rawLim = inLevels[curLevel + 1]; + final int rawPop = rawLim - rawBeg; + + if ((currentItemCount < targetItemCount) || (rawPop < KllHelper.levelCapacity(k, numLevels, curLevel, m))) { + // copy level over as is + // because inBuf and outBuf could be the same, make sure we are not moving data upwards! + assert (rawBeg >= outLevels[curLevel]); + System.arraycopy(inBuf, rawBeg, outBuf, outLevels[curLevel], rawPop); + outLevels[curLevel + 1] = outLevels[curLevel] + rawPop; + } + else { + // The sketch is too full AND this level is too full, so we compact it + // Note: this can add a level and thus change the sketch's capacity + + final int popAbove = inLevels[curLevel + 2] - rawLim; + final boolean oddPop = isOdd(rawPop); + final int adjBeg = oddPop ? 1 + rawBeg : rawBeg; + final int adjPop = oddPop ? rawPop - 1 : rawPop; + final int halfAdjPop = adjPop / 2; + + if (oddPop) { // copy one guy over + outBuf[outLevels[curLevel]] = inBuf[rawBeg]; + outLevels[curLevel + 1] = outLevels[curLevel] + 1; + } else { // copy zero guys over + outLevels[curLevel + 1] = outLevels[curLevel]; + } + + // level zero might not be sorted, so we must sort it if we wish to compact it + if ((curLevel == 0) && !isLevelZeroSorted) { + Arrays.sort(inBuf, adjBeg, adjBeg + adjPop); + } + + if (popAbove == 0) { // Level above is empty, so halve up + randomlyHalveUpLongs(inBuf, adjBeg, adjPop, random); + } else { // Level above is nonempty, so halve down, then merge up + randomlyHalveDownLongs(inBuf, adjBeg, adjPop, random); + mergeSortedLongArrays(inBuf, adjBeg, halfAdjPop, inBuf, rawLim, popAbove, inBuf, adjBeg + halfAdjPop); + } + + // track the fact that we just eliminated some data + currentItemCount -= halfAdjPop; + + // Adjust the boundaries of the level above + inLevels[curLevel + 1] = inLevels[curLevel + 1] - halfAdjPop; + + // Increment numLevels if we just compacted the old top level + // This creates some more capacity (the size of the new bottom level) + if (curLevel == (numLevels - 1)) { + numLevels++; + targetItemCount += KllHelper.levelCapacity(k, numLevels, 0, m); + } + } // end of code for compacting a level + + // determine whether we have processed all levels yet (including any new levels that we created) + if (curLevel == (numLevels - 1)) { doneYet = true; } + } // end of loop over levels + + assert (outLevels[numLevels] - outLevels[0]) == currentItemCount; + return new int[] {numLevels, targetItemCount, currentItemCount}; + } + + private static void populateLongWorkArrays( //workBuf and workLevels are modified + final long[] workBuf, final int[] workLevels, final int provisionalNumLevels, + final int myCurNumLevels, final int[] myCurLevelsArr, final long[] myCurLongItemsArr, + final int otherNumLevels, final int[] otherLevelsArr, final long[] otherLongItemsArr) { + + workLevels[0] = 0; + + // Note: the level zero data from "other" was already inserted into "self". + // This copies into workbuf. + final int selfPopZero = KllHelper.currentLevelSizeItems(0, myCurNumLevels, myCurLevelsArr); + System.arraycopy(myCurLongItemsArr, myCurLevelsArr[0], workBuf, workLevels[0], selfPopZero); + workLevels[1] = workLevels[0] + selfPopZero; + + for (int lvl = 1; lvl < provisionalNumLevels; lvl++) { + final int selfPop = KllHelper.currentLevelSizeItems(lvl, myCurNumLevels, myCurLevelsArr); + final int otherPop = KllHelper.currentLevelSizeItems(lvl, otherNumLevels, otherLevelsArr); + workLevels[lvl + 1] = workLevels[lvl] + selfPop + otherPop; + assert selfPop >= 0 && otherPop >= 0; + if (selfPop == 0 && otherPop == 0) { continue; } + if (selfPop > 0 && otherPop == 0) { + System.arraycopy(myCurLongItemsArr, myCurLevelsArr[lvl], workBuf, workLevels[lvl], selfPop); + } + else if (selfPop == 0 && otherPop > 0) { + System.arraycopy(otherLongItemsArr, otherLevelsArr[lvl], workBuf, workLevels[lvl], otherPop); + } + else if (selfPop > 0 && otherPop > 0) { + mergeSortedLongArrays( //only workBuf is modified + myCurLongItemsArr, myCurLevelsArr[lvl], selfPop, + otherLongItemsArr, otherLevelsArr[lvl], otherPop, + workBuf, workLevels[lvl]); + } + } + } + + /* + * Validation Method. + * The following must be enabled for use with the KllDoublesValidationTest, + * which is only enabled for manual testing. In addition, two Validation Methods + * above need to be modified. + */ //NOTE Validation Method: Need to uncomment to use + // static int nextOffset = 0; + // + // private static int deterministicOffset() { + // final int result = nextOffset; + // nextOffset = 1 - nextOffset; + // return result; + // } + +} diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java new file mode 100644 index 000000000..0f6fa7d89 --- /dev/null +++ b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java @@ -0,0 +1,669 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import static java.lang.Math.max; +import static java.lang.Math.min; +import static org.apache.datasketches.common.ByteArrayUtil.putLongLE; +import static org.apache.datasketches.kll.KllSketch.SketchStructure.UPDATABLE; +import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH; + +import java.util.Arrays; +import java.util.Objects; + +import org.apache.datasketches.common.ArrayOfItemsSerDe; +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.common.SuppressFBWarnings; +import org.apache.datasketches.kll.KllDirectLongsSketch.KllDirectCompactLongsSketch; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.MemoryRequestServer; +import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.quantilescommon.LongsSketchSortedView; +import org.apache.datasketches.quantilescommon.QuantileSearchCriteria; +import org.apache.datasketches.quantilescommon.QuantilesLongsAPI; +import org.apache.datasketches.quantilescommon.QuantilesLongsSketchIterator; + +/** + * This variation of the KllSketch implements primitive longs. + * + * @see org.apache.datasketches.kll.KllSketch + */ +public abstract class KllLongsSketch extends KllSketch implements QuantilesLongsAPI { + private LongsSketchSortedView longsSV = null; + final static int ITEM_BYTES = Long.BYTES; + + KllLongsSketch( + final SketchStructure sketchStructure) { + super(SketchType.LONGS_SKETCH, sketchStructure); + } + + //Factories for new heap instances. + + /** + * Create a new heap instance of this sketch with the default k = 200. + * The default k = 200 results in a normalized rank error of about + * 1.65%. Larger K will have smaller error but the sketch will be larger (and slower). + * @return new KllLongsSketch on the Java heap. + */ + public static KllLongsSketch newHeapInstance() { + return newHeapInstance(DEFAULT_K); + } + + /** + * Create a new heap instance of this sketch with a given parameter k. + * k can be between 8, inclusive, and 65535, inclusive. + * The default k = 200 results in a normalized rank error of about + * 1.65%. Larger K will have smaller error but the sketch will be larger (and slower). + * @param k parameter that controls size of the sketch and accuracy of estimates. + * @return new KllLongsSketch on the Java heap. + */ + public static KllLongsSketch newHeapInstance(final int k) { + return new KllHeapLongsSketch(k, DEFAULT_M); + } + + //Factories for new direct instances. + + /** + * Create a new direct updatable instance of this sketch with the default k. + * The default k = 200 results in a normalized rank error of about + * 1.65%. Larger k will have smaller error but the sketch will be larger (and slower). + * @param dstMem the given destination WritableMemory object for use by the sketch + * @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory + * @return a new direct instance of this sketch + */ + public static KllLongsSketch newDirectInstance( + final WritableMemory dstMem, + final MemoryRequestServer memReqSvr) { + return newDirectInstance(DEFAULT_K, dstMem, memReqSvr); + } + + /** + * Create a new direct updatable instance of this sketch with a given k. + * @param k parameter that controls size of the sketch and accuracy of estimates. + * @param dstMem the given destination WritableMemory object for use by the sketch + * @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory + * @return a new direct instance of this sketch + */ + public static KllLongsSketch newDirectInstance( + final int k, + final WritableMemory dstMem, + final MemoryRequestServer memReqSvr) { + Objects.requireNonNull(dstMem, "Parameter 'dstMem' must not be null"); + Objects.requireNonNull(memReqSvr, "Parameter 'memReqSvr' must not be null"); + return KllDirectLongsSketch.newDirectUpdatableInstance(k, DEFAULT_M, dstMem, memReqSvr); + } + + //Factory to create an heap instance from a Memory image + + /** + * Factory heapify takes a compact sketch image in Memory and instantiates an on-heap sketch. + * The resulting sketch will not retain any link to the source Memory. + * @param srcMem a compact Memory image of a sketch serialized by this sketch. + * See Memory + * @return a heap-based sketch based on the given Memory. + */ + public static KllLongsSketch heapify(final Memory srcMem) { + Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null"); + return KllHeapLongsSketch.heapifyImpl(srcMem); + } + + //Factory to wrap a Read-Only Memory + + /** + * Wrap a sketch around the given read only compact source Memory containing sketch data + * that originated from this sketch. + * @param srcMem the read only source Memory + * @return instance of this sketch + */ + public static KllLongsSketch wrap(final Memory srcMem) { + Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null"); + final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, LONGS_SKETCH, null); + if (memVal.sketchStructure == UPDATABLE) { + final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); //dummy + return new KllDirectLongsSketch(memVal.sketchStructure, (WritableMemory)srcMem, memReqSvr, memVal); + } else { + return new KllDirectCompactLongsSketch(memVal.sketchStructure, srcMem, memVal); + } + } + + //Factory to wrap a WritableMemory image + + /** + * Wrap a sketch around the given source Writable Memory containing sketch data + * that originated from this sketch. + * @param srcMem a WritableMemory that contains data. + * @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory + * @return instance of this sketch + */ + public static KllLongsSketch writableWrap( + final WritableMemory srcMem, + final MemoryRequestServer memReqSvr) { + Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null"); + Objects.requireNonNull(memReqSvr, "Parameter 'memReqSvr' must not be null"); + final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, LONGS_SKETCH); + if (memVal.sketchStructure == UPDATABLE) { + return new KllDirectLongsSketch(UPDATABLE, srcMem, memReqSvr, memVal); + } else { + return new KllDirectCompactLongsSketch(memVal.sketchStructure, srcMem, memVal); + } + } + + //END of Constructors + + @Override + public double[] getCDF(final long[] splitPoints, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + refreshSortedView(); + return longsSV.getCDF(splitPoints, searchCrit); + } + + @Override + public double[] getPMF(final long[] splitPoints, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + refreshSortedView(); + return longsSV.getPMF(splitPoints, searchCrit); + } + + @Override + public long getQuantile(final double rank, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + refreshSortedView(); + return longsSV.getQuantile(rank, searchCrit); + } + + @Override + public long[] getQuantiles(final double[] ranks, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + refreshSortedView(); + final int len = ranks.length; + final long[] quantiles = new long[len]; + for (int i = 0; i < len; i++) { + quantiles[i] = longsSV.getQuantile(ranks[i], searchCrit); + } + return quantiles; + } + + /** + * {@inheritDoc} + * The approximate probability that the true quantile is within the confidence interval + * specified by the upper and lower quantile bounds for this sketch is 0.99. + */ + @Override + public long getQuantileLowerBound(final double rank) { + return getQuantile(max(0, rank - KllHelper.getNormalizedRankError(getMinK(), false))); + } + + /** + * {@inheritDoc} + * The approximate probability that the true quantile is within the confidence interval + * specified by the upper and lower quantile bounds for this sketch is 0.99. + */ + @Override + public long getQuantileUpperBound(final double rank) { + return getQuantile(min(1.0, rank + KllHelper.getNormalizedRankError(getMinK(), false))); + } + + @Override + public double getRank(final long quantile, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + refreshSortedView(); + return longsSV.getRank(quantile, searchCrit); + } + + /** + * {@inheritDoc} + * The approximate probability that the true rank is within the confidence interval + * specified by the upper and lower rank bounds for this sketch is 0.99. + */ + @Override + public double getRankLowerBound(final double rank) { + return max(0.0, rank - KllHelper.getNormalizedRankError(getMinK(), false)); + } + + /** + * {@inheritDoc} + * The approximate probability that the true rank is within the confidence interval + * specified by the upper and lower rank bounds for this sketch is 0.99. + */ + @Override + public double getRankUpperBound(final double rank) { + return min(1.0, rank + KllHelper.getNormalizedRankError(getMinK(), false)); + } + + @Override + public double[] getRanks(final long[] quantiles, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + refreshSortedView(); + final int len = quantiles.length; + final double[] ranks = new double[len]; + for (int i = 0; i < len; i++) { + ranks[i] = longsSV.getRank(quantiles[i], searchCrit); + } + return ranks; + } + + @Override + public QuantilesLongsSketchIterator iterator() { + return new KllLongsSketchIterator( + getLongItemsArray(), getLevelsArray(SketchStructure.UPDATABLE), getNumLevels()); + } + + @Override + public final void merge(final KllSketch other) { + if (readOnly || sketchStructure != UPDATABLE) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + if (this == other) { throw new SketchesArgumentException(SELF_MERGE_MSG); } + final KllLongsSketch otherLngSk = (KllLongsSketch)other; + if (otherLngSk.isEmpty()) { return; } + KllLongsHelper.mergeLongsImpl(this, otherLngSk); + longsSV = null; + } + + /** + * {@inheritDoc} + *

The parameter k will not change.

+ */ + @Override + public final void reset() { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + final int k = getK(); + setN(0); + setMinK(k); + setNumLevels(1); + setLevelZeroSorted(false); + setLevelsArray(new int[] {k, k}); + setMinItem(Long.MAX_VALUE); + setMaxItem(Long.MIN_VALUE); + setLongItemsArray(new long[k]); + longsSV = null; + } + + @Override + public byte[] toByteArray() { + return KllHelper.toByteArray(this, false); + } + + @Override + public String toString(final boolean withLevels, final boolean withLevelsAndItems) { + KllSketch sketch = this; + if (withLevelsAndItems && sketchStructure != UPDATABLE) { + final Memory mem = getWritableMemory(); + assert mem != null; + sketch = KllLongsSketch.heapify(getWritableMemory()); + } + return KllHelper.toStringImpl(sketch, withLevels, withLevelsAndItems, getSerDe()); + } + + //SINGLE UPDATE + + @Override + public void update(final long item) { + // Align with KllDoublesSketch + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + updateLong(this, item); + longsSV = null; + } + + //Also Called from KllLongsHelper::merge + static void updateLong(final KllLongsSketch lngSk, final long item) { + lngSk.updateMinMax(item); + int freeSpace = lngSk.levelsArr[0]; + assert (freeSpace >= 0); + if (freeSpace == 0) { + KllLongsHelper.compressWhileUpdatingSketch(lngSk); + freeSpace = lngSk.levelsArr[0]; + assert (freeSpace > 0); + } + lngSk.incN(1); + lngSk.setLevelZeroSorted(false); + final int nextPos = freeSpace - 1; + lngSk.setLevelsArrayAt(0, nextPos); + lngSk.setLongItemsArrayAt(nextPos, item); + } + + /** + * Single update of min and max + * @param item the source item, it must not be a NaN. + */ + final void updateMinMax(final long item) { + if (isEmpty()) { + setMinItem(item); + setMaxItem(item); + } else { + setMinItem(min(getMinItemInternal(), item)); + setMaxItem(max(getMaxItemInternal(), item)); + } + } + + //WEIGHTED UPDATE + + /** + * Weighted update. Updates this sketch with the given item the number of times specified by the given integer weight. + * @param item the item to be repeated. NaNs are ignored. + * @param weight the number of times the update of item is to be repeated. It must be ≥ one. + */ + public void update(final long item, final long weight) { + // + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + if (weight < 1L) { throw new SketchesArgumentException("Weight is less than one."); } + if (weight == 1L) { updateLong(this, item); } + else { + if (weight < levelsArr[0]) { + for (int i = 0; i < (int)weight; i++) { updateLong(this, item); } + } else { + final KllHeapLongsSketch tmpSk = new KllHeapLongsSketch(getK(), DEFAULT_M, item, weight); + merge(tmpSk); + } + } + longsSV = null; + } + + // VECTOR UPDATE + + /** + * Vector update. Updates this sketch with the given array (vector) of items, starting at the items + * offset for a length number of items. This is not supported for direct sketches. + * @param items the vector of items + * @param offset the starting index of the items[] array + * @param length the number of items + */ + public void update(final long[] items, final int offset, final int length) { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + if (length == 0) { return; } + updateLong(items, offset, length); + longsSV = null; + } + /* Align with KllDoublesSketch + + + + + + + + + + + + + */ + private void updateLong(final long[] srcItems, final int srcOffset, final int length) { + if (isEmpty()) { + setMinItem(srcItems[srcOffset]); //initialize with a real value + setMaxItem(srcItems[srcOffset]); + } + + int count = 0; + while (count < length) { + if (levelsArr[0] == 0) { + KllLongsHelper.compressWhileUpdatingSketch(this); + } + final int spaceNeeded = length - count; + final int freeSpace = levelsArr[0]; + assert (freeSpace > 0); + final int numItemsToCopy = min(spaceNeeded, freeSpace); + final int dstOffset = freeSpace - numItemsToCopy; + final int localSrcOffset = srcOffset + count; + setLongItemsArrayAt(dstOffset, srcItems, localSrcOffset, numItemsToCopy); + updateMinMax(srcItems, localSrcOffset, numItemsToCopy); + count += numItemsToCopy; + incN(numItemsToCopy); + setLevelsArrayAt(0, dstOffset); + } + setLevelZeroSorted(false); + } + + /** + * Vector update of min and max. + * @param srcItems the input source array of values, no NaNs allowed. + * @param srcOffset the starting offset in srcItems + * @param length the number of items to update min and max + */ + private void updateMinMax(final long[] srcItems, final int srcOffset, final int length) { + final int end = srcOffset + length; + for (int i = srcOffset; i < end; i++) { + setMinItem(min(getMinItemInternal(), srcItems[i])); + setMaxItem(max(getMaxItemInternal(), srcItems[i])); + } + } + /* Align with KllDoublesSketch + + + + + + + + + */ + // END ALL UPDATE METHODS + + /** + * @return full size of internal items array including empty space at bottom. + */ + abstract long[] getLongItemsArray(); + + /** + * @return items array of retained items. + */ + abstract long[] getLongRetainedItemsArray(); + + abstract long getLongSingleItem(); + + // Min & Max Methods + + abstract long getMaxItemInternal(); + + abstract void setMaxItem(long item); + + abstract long getMinItemInternal(); + + abstract void setMinItem(long item); + + @Override + abstract byte[] getMinMaxByteArr(); + + @Override + int getMinMaxSizeBytes() { + return Long.BYTES * 2; + } + + //END Min & Max Methods + + @Override + abstract byte[] getRetainedItemsByteArr(); + + @Override + int getRetainedItemsSizeBytes() { + return getNumRetained() * Long.BYTES; + } + + @Override + ArrayOfItemsSerDe getSerDe() { return null; } + + @Override + final byte[] getSingleItemByteArr() { + final byte[] bytes = new byte[ITEM_BYTES]; + putLongLE(bytes, 0, getLongSingleItem()); + return bytes; + } + + @Override + int getSingleItemSizeBytes() { + return Long.BYTES; + } + + @Override + abstract byte[] getTotalItemsByteArr(); + + @Override + int getTotalItemsNumBytes() { + return levelsArr[getNumLevels()] * Long.BYTES; + } + + abstract void setLongItemsArray(long[] longItems); + + abstract void setLongItemsArrayAt(int index, long item); + + abstract void setLongItemsArrayAt(int dstIndex, long[] srcItems, int srcOffset, int length); + + // SORTED VIEW + + @Override + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "OK in this case.") + public LongsSketchSortedView getSortedView() { + refreshSortedView(); + return longsSV; + } + + private final LongsSketchSortedView refreshSortedView() { + if (longsSV == null) { + final CreateSortedView csv = new CreateSortedView(); + longsSV = csv.getSV(); + } + return longsSV; + } + + private final class CreateSortedView { + long[] quantiles; + long[] cumWeights; + + LongsSketchSortedView getSV() { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + final long[] srcQuantiles = getLongItemsArray(); + final int[] srcLevels = levelsArr; + final int srcNumLevels = getNumLevels(); + + if (!isLevelZeroSorted()) { + Arrays.sort(srcQuantiles, srcLevels[0], srcLevels[1]); + if (!hasMemory()) { setLevelZeroSorted(true); } + //we don't sort level0 in Memory, only our copy. + } + final int numQuantiles = getNumRetained(); + quantiles = new long[numQuantiles]; + cumWeights = new long[numQuantiles]; + populateFromSketch(srcQuantiles, srcLevels, srcNumLevels, numQuantiles); + return new LongsSketchSortedView( + quantiles, cumWeights, KllLongsSketch.this); + } + + private void populateFromSketch(final long[] srcQuantiles, final int[] srcLevels, + final int srcNumLevels, final int numItems) { + final int[] myLevels = new int[srcNumLevels + 1]; + final int offset = srcLevels[0]; + System.arraycopy(srcQuantiles, offset, quantiles, 0, numItems); + int srcLevel = 0; + int dstLevel = 0; + long weight = 1; + while (srcLevel < srcNumLevels) { + final int fromIndex = srcLevels[srcLevel] - offset; + final int toIndex = srcLevels[srcLevel + 1] - offset; // exclusive + if (fromIndex < toIndex) { // if equal, skip empty level + Arrays.fill(cumWeights, fromIndex, toIndex, weight); + myLevels[dstLevel] = fromIndex; + myLevels[dstLevel + 1] = toIndex; + dstLevel++; + } + srcLevel++; + weight *= 2; + } + final int numLevels = dstLevel; + blockyTandemMergeSort(quantiles, cumWeights, myLevels, numLevels); //create unit weights + KllHelper.convertToCumulative(cumWeights); + } + } //End of class CreateSortedView + + private static void blockyTandemMergeSort(final long[] quantiles, final long[] weights, + final int[] levels, final int numLevels) { + if (numLevels == 1) { return; } + + // duplicate the input in preparation for the "ping-pong" copy reduction strategy. + final long[] quantilesTmp = Arrays.copyOf(quantiles, quantiles.length); + final long[] weightsTmp = Arrays.copyOf(weights, quantiles.length); // don't need the extra one + + blockyTandemMergeSortRecursion(quantilesTmp, weightsTmp, quantiles, weights, levels, 0, numLevels); + } + + private static void blockyTandemMergeSortRecursion( + final long[] quantilesSrc, final long[] weightsSrc, + final long[] quantilesDst, final long[] weightsDst, + final int[] levels, final int startingLevel, final int numLevels) { + if (numLevels == 1) { return; } + final int numLevels1 = numLevels / 2; + final int numLevels2 = numLevels - numLevels1; + assert numLevels1 >= 1; + assert numLevels2 >= numLevels1; + final int startingLevel1 = startingLevel; + final int startingLevel2 = startingLevel + numLevels1; + // swap roles of src and dst + blockyTandemMergeSortRecursion( + quantilesDst, weightsDst, + quantilesSrc, weightsSrc, + levels, startingLevel1, numLevels1); + blockyTandemMergeSortRecursion( + quantilesDst, weightsDst, + quantilesSrc, weightsSrc, + levels, startingLevel2, numLevels2); + tandemMerge( + quantilesSrc, weightsSrc, + quantilesDst, weightsDst, + levels, + startingLevel1, numLevels1, + startingLevel2, numLevels2); + } + + private static void tandemMerge( + final long[] quantilesSrc, final long[] weightsSrc, + final long[] quantilesDst, final long[] weightsDst, + final int[] levelStarts, + final int startingLevel1, final int numLevels1, + final int startingLevel2, final int numLevels2) { + final int fromIndex1 = levelStarts[startingLevel1]; + final int toIndex1 = levelStarts[startingLevel1 + numLevels1]; // exclusive + final int fromIndex2 = levelStarts[startingLevel2]; + final int toIndex2 = levelStarts[startingLevel2 + numLevels2]; // exclusive + int iSrc1 = fromIndex1; + int iSrc2 = fromIndex2; + int iDst = fromIndex1; + + while (iSrc1 < toIndex1 && iSrc2 < toIndex2) { + if (quantilesSrc[iSrc1] < quantilesSrc[iSrc2]) { + quantilesDst[iDst] = quantilesSrc[iSrc1]; + weightsDst[iDst] = weightsSrc[iSrc1]; + iSrc1++; + } else { + quantilesDst[iDst] = quantilesSrc[iSrc2]; + weightsDst[iDst] = weightsSrc[iSrc2]; + iSrc2++; + } + iDst++; + } + if (iSrc1 < toIndex1) { + System.arraycopy(quantilesSrc, iSrc1, quantilesDst, iDst, toIndex1 - iSrc1); + System.arraycopy(weightsSrc, iSrc1, weightsDst, iDst, toIndex1 - iSrc1); + } else if (iSrc2 < toIndex2) { + System.arraycopy(quantilesSrc, iSrc2, quantilesDst, iDst, toIndex2 - iSrc2); + System.arraycopy(weightsSrc, iSrc2, weightsDst, iDst, toIndex2 - iSrc2); + } + } + + // END SORTED VIEW + +} diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsSketchIterator.java b/src/main/java/org/apache/datasketches/kll/KllLongsSketchIterator.java new file mode 100644 index 000000000..8922ec70e --- /dev/null +++ b/src/main/java/org/apache/datasketches/kll/KllLongsSketchIterator.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import org.apache.datasketches.quantilescommon.QuantilesLongsSketchIterator; + +/** + * Iterator over KllLongsSketch. The order is not defined. + */ +public final class KllLongsSketchIterator extends KllSketchIterator implements QuantilesLongsSketchIterator { + private final long[] quantiles; + + KllLongsSketchIterator(final long[] quantiles, final int[] levelsArr, final int numLevels) { + super(levelsArr, numLevels); + this.quantiles = quantiles; + } + + @Override + public long getQuantile() { + return quantiles[index]; + } + +} diff --git a/src/main/java/org/apache/datasketches/quantilescommon/LongsSketchSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/LongsSketchSortedView.java new file mode 100644 index 000000000..efb4006f6 --- /dev/null +++ b/src/main/java/org/apache/datasketches/quantilescommon/LongsSketchSortedView.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantilescommon; + +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; +import static org.apache.datasketches.quantilescommon.QuantilesAPI.EMPTY_MSG; +import static org.apache.datasketches.quantilescommon.QuantilesUtil.getNaturalRank; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.quantilescommon.IncludeMinMax.LongsPair; + +/** + * The SortedView of the KllLongsSketch. + * @author Lee Rhodes + * @author Zac Blanco + */ +public final class LongsSketchSortedView implements LongsSortedView { + private final long[] quantiles; + private final long[] cumWeights; //cumulative natural weights + private final long totalN; + + /** + * Construct from elements, also used in testing. + * @param quantiles sorted array of quantiles + * @param cumWeights sorted, monotonically increasing cumulative weights. + * @param sk the underlying quantile sketch. + */ + public LongsSketchSortedView( + final long[] quantiles, + final long[] cumWeights, + final QuantilesLongsAPI sk) { + final LongsPair dPair = + IncludeMinMax.includeLongsMinMax(quantiles, cumWeights, sk.getMaxItem(), sk.getMinItem()); + this.quantiles = dPair.quantiles; + this.cumWeights = dPair.cumWeights; + this.totalN = sk.getN(); + } + + //Used for testing + LongsSketchSortedView( + final long[] quantiles, + final long[] cumWeights, + final long totalN, + final long maxItem, + final long minItem) { + final LongsPair dPair = + IncludeMinMax.includeLongsMinMax(quantiles, cumWeights, maxItem, minItem); + this.quantiles = dPair.quantiles; + this.cumWeights = dPair.cumWeights; + this.totalN = totalN; + } + + @Override + public long[] getCumulativeWeights() { + return cumWeights.clone(); + } + + @Override + public long getMaxItem() { + final int top = quantiles.length - 1; + return quantiles[top]; + } + + @Override + public long getMinItem() { + return quantiles[0]; + } + + @Override + public long getN() { + return totalN; + } + + @Override + public int getNumRetained() { + return quantiles.length; + } + + @Override + public long getQuantile(final double rank, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + QuantilesUtil.checkNormalizedRankBounds(rank); + final int len = cumWeights.length; + final double naturalRank = getNaturalRank(rank, totalN, searchCrit); + final InequalitySearch crit = (searchCrit == INCLUSIVE) ? InequalitySearch.GE : InequalitySearch.GT; + final int index = InequalitySearch.find(cumWeights, 0, len - 1, naturalRank, crit); + if (index == -1) { + return quantiles[len - 1]; //EXCLUSIVE (GT) case: normRank == 1.0; + } + return quantiles[index]; + } + + @Override + public long[] getQuantiles() { + return quantiles.clone(); + } + + @Override + public double getRank(final long quantile, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + final int len = quantiles.length; + final InequalitySearch crit = (searchCrit == INCLUSIVE) ? InequalitySearch.LE : InequalitySearch.LT; + final int index = InequalitySearch.find(quantiles, 0, len - 1, quantile, crit); + if (index == -1) { + return 0; //EXCLUSIVE (LT) case: quantile <= minQuantile; INCLUSIVE (LE) case: quantile < minQuantile + } + return (double)cumWeights[index] / totalN; + } + + @Override + public boolean isEmpty() { + return totalN == 0; + } + + @Override + public LongsSortedViewIterator iterator() { + return new LongsSortedViewIterator(quantiles, cumWeights); + } + +} diff --git a/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java new file mode 100644 index 000000000..4823edd1d --- /dev/null +++ b/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantilescommon; + +/** + * The Sorted View for quantile sketches of primitive type long. + * @see SortedView + * @author Lee Rhodes + * @author Zac Blanco + */ +public interface LongsSortedView extends SortedView { + + /** + * Returns an approximation to the Cumulative Distribution Function (CDF) of the input stream + * as a monotonically increasing array of double ranks (or cumulative probabilities) on the interval [0.0, 1.0], + * given a set of splitPoints. + * + *

The resulting approximations have a probabilistic guarantee that can be obtained from the + * getNormalizedRankError(false) function.

+ * + * @param splitPoints an array of m unique, monotonically increasing items + * (of the same type as the input items) + * that divide the item input domain into m+1 overlapping intervals. + * + *

The start of each interval is below the lowest item retained by the sketch + * corresponding to a zero rank or zero probability, and the end of the interval + * is the rank or cumulative probability corresponding to the split point.

+ * + *

The (m+1)th interval represents 100% of the distribution represented by the sketch + * and consistent with the definition of a cumulative probability distribution, thus the (m+1)th + * rank or probability in the returned array is always 1.0.

+ * + *

If a split point exactly equals a retained item of the sketch and the search criterion is:

+ * + *
    + *
  • INCLUSIVE, the resulting cumulative probability will include that item.
    • + *
  • EXCLUSIVE, the resulting cumulative probability will not include the weight of that split point.
    • + *
+ * + *

It is not recommended to include either the minimum or maximum items of the input stream.

+ * + * @param searchCrit the desired search criteria. + * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. + * @throws IllegalArgumentException if sketch is empty. + */ + default double[] getCDF(long[] splitPoints, QuantileSearchCriteria searchCrit) { + QuantilesUtil.checkLongsSplitPointsOrder(splitPoints); + final int len = splitPoints.length + 1; + final double[] buckets = new double[len]; + for (int i = 0; i < len - 1; i++) { + buckets[i] = getRank(splitPoints[i], searchCrit); + } + buckets[len - 1] = 1; + return buckets; + } + + /** + * Returns the maximum item of the stream. This may be distinct from the largest item retained by the + * sketch algorithm. + * + * @return the maximum item of the stream + * @throws IllegalArgumentException if sketch is empty. + */ + long getMaxItem(); + + /** + * Returns the minimum item of the stream. This may be distinct from the smallest item retained by the + * sketch algorithm. + * + * @return the minimum item of the stream + * @throws IllegalArgumentException if sketch is empty. + */ + long getMinItem(); + + /** + * Returns an approximation to the Probability Mass Function (PMF) of the input stream + * as an array of probability masses as doubles on the interval [0.0, 1.0], + * given a set of splitPoints. + * + *

The resulting approximations have a probabilistic guarantee that can be obtained from the + * getNormalizedRankError(true) function.

+ * + * @param splitPoints an array of m unique, monotonically increasing items + * (of the same type as the input items) + * that divide the item input domain into m+1 consecutive, non-overlapping intervals. + * + *

Each interval except for the end intervals starts with a split point and ends with the next split + * point in sequence.

+ * + *

The first interval starts below the lowest item retained by the sketch + * corresponding to a zero rank or zero probability, and ends with the first split point

+ * + *

The last (m+1)th interval starts with the last split point and ends after the last + * item retained by the sketch corresponding to a rank or probability of 1.0.

+ * + *

The sum of the probability masses of all (m+1) intervals is 1.0.

+ * + *

If the search criterion is:

+ * + *
    + *
  • INCLUSIVE, and the upper split point of an interval equals an item retained by the sketch, the interval + * will include that item. If the lower split point equals an item retained by the sketch, the interval will exclude + * that item.
    • + *
  • EXCLUSIVE, and the upper split point of an interval equals an item retained by the sketch, the interval + * will exclude that item. If the lower split point equals an item retained by the sketch, the interval will include + * that item.
    • + *
+ * + *

It is not recommended to include either the minimum or maximum items of the input stream.

+ * + * @param searchCrit the desired search criteria. + * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. + * @throws IllegalArgumentException if sketch is empty. + */ + default double[] getPMF(long[] splitPoints, QuantileSearchCriteria searchCrit) { + final double[] buckets = getCDF(splitPoints, searchCrit); + final int len = buckets.length; + for (int i = len; i-- > 1; ) { + buckets[i] -= buckets[i - 1]; + } + return buckets; + } + + /** + * Gets the approximate quantile of the given normalized rank and the given search criterion. + * + * @param rank the given normalized rank, a double in the range [0.0, 1.0]. + * @param searchCrit If INCLUSIVE, the given rank includes all quantiles ≤ + * the quantile directly corresponding to the given rank. + * If EXCLUSIVE, he given rank includes all quantiles < + * the quantile directly corresponding to the given rank. + * @return the approximate quantile given the normalized rank. + * @throws IllegalArgumentException if sketch is empty. + * @see QuantileSearchCriteria + */ + long getQuantile(double rank, QuantileSearchCriteria searchCrit); + + /** + * Returns an array of all retained quantiles by the sketch. + * @return an array of all retained quantiles by the sketch. + */ + long[] getQuantiles(); + + /** + * Gets the normalized rank corresponding to the given a quantile. + * + * @param quantile the given quantile + * @param searchCrit if INCLUSIVE the given quantile is included into the rank. + * @return the normalized rank corresponding to the given quantile. + * @throws IllegalArgumentException if sketch is empty. + * @see QuantileSearchCriteria + */ + double getRank(long quantile, QuantileSearchCriteria searchCrit); + + @Override + LongsSortedViewIterator iterator(); + +} diff --git a/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedViewIterator.java b/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedViewIterator.java new file mode 100644 index 000000000..77510cd5a --- /dev/null +++ b/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedViewIterator.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantilescommon; + +/** + * Iterator over quantile sketches of primitive type long. + */ +public final class LongsSortedViewIterator extends SortedViewIterator { + private final long[] quantiles; + + /** + * Constructor. + * @param quantiles the given array of quantiles, which must be ordered. + * @param cumWeights the given array of cumulative weights, which must be ordered, start with the value one, and + * the last value must be equal to N, the total number of items updated to the sketch. + */ + public LongsSortedViewIterator(final long[] quantiles, final long[] cumWeights) { + super(cumWeights); + this.quantiles = quantiles; //SpotBugs EI_EXPOSE_REP2 suppressed by FindBugsExcludeFilter + } + + /** + * Gets the quantile at the current index. + * + *

Don't call this before calling next() for the first time + * or after getting false from next().

+ * + * @return the quantile at the current index. + */ + public long getQuantile() { + return quantiles[index]; + } + +} diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java new file mode 100644 index 000000000..2b542a68f --- /dev/null +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantilescommon; + +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; + +/** + * The Quantiles API for item type long. + * @see QuantilesAPI + * @author Lee Rhodes + * @author Zac Blanco + */ +public interface QuantilesLongsAPI extends QuantilesAPI { + + /** + * This is equivalent to {@link #getCDF(long[], QuantileSearchCriteria) getCDF(splitPoints, INCLUSIVE)} + * @param splitPoints an array of m unique, monotonically increasing items. + * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. + * @throws IllegalArgumentException if sketch is empty. + */ + default double[] getCDF(long[] splitPoints) { + return getCDF(splitPoints, INCLUSIVE); + } + + /** + * Returns an approximation to the Cumulative Distribution Function (CDF) of the input stream + * as a monotonically increasing array of double ranks (or cumulative probabilities) on the interval [0.0, 1.0], + * given a set of splitPoints. + * + *

The resulting approximations have a probabilistic guarantee that can be obtained from the + * getNormalizedRankError(false) function.

+ * + * @param splitPoints an array of m unique, monotonically increasing items + * (of the same type as the input items) + * that divide the item input domain into m+1 overlapping intervals. + * + *

The start of each interval is below the lowest item retained by the sketch + * corresponding to a zero rank or zero probability, and the end of the interval + * is the rank or cumulative probability corresponding to the split point.

+ * + *

The (m+1)th interval represents 100% of the distribution represented by the sketch + * and consistent with the definition of a cumulative probability distribution, thus the (m+1)th + * rank or probability in the returned array is always 1.0.

+ * + *

If a split point exactly equals a retained item of the sketch and the search criterion is:

+ * + *
    + *
  • INCLUSIVE, the resulting cumulative probability will include that item.
    • + *
  • EXCLUSIVE, the resulting cumulative probability will not include the weight of that split point.
    • + *
+ * + *

It is not recommended to include either the minimum or maximum items of the input stream.

+ * + * @param searchCrit the desired search criteria. + * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. + * @throws IllegalArgumentException if sketch is empty. + */ + double[] getCDF(long[] splitPoints, QuantileSearchCriteria searchCrit); + + /** + * Returns the maximum item of the stream. This is provided for convenience and may be different from the + * item returned by getQuantile(1.0). + * + * @return the maximum item of the stream + * @throws IllegalArgumentException if sketch is empty. + */ + long getMaxItem(); + + /** + * Returns the minimum item of the stream. This is provided for convenience and may be different from the + * item returned by getQuantile(0.0). + * + * @return the minimum item of the stream + * @throws IllegalArgumentException if sketch is empty. + */ + long getMinItem(); + + /** + * This is equivalent to {@link #getPMF(long[], QuantileSearchCriteria) getPMF(splitPoints, INCLUSIVE)} + * @param splitPoints an array of m unique, monotonically increasing items. + * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. + * @throws IllegalArgumentException if sketch is empty. + */ + default double[] getPMF(long[] splitPoints) { + return getPMF(splitPoints, INCLUSIVE); + } + + /** + * Returns an approximation to the Probability Mass Function (PMF) of the input stream + * as an array of probability masses as doubles on the interval [0.0, 1.0], + * given a set of splitPoints. + * + *

The resulting approximations have a probabilistic guarantee that can be obtained from the + * getNormalizedRankError(true) function.

+ * + * @param splitPoints an array of m unique, monotonically increasing items + * (of the same type as the input items) + * that divide the item input domain into m+1 consecutive, non-overlapping intervals. + * + *

Each interval except for the end intervals starts with a split point and ends with the next split + * point in sequence.

+ * + *

The first interval starts below the lowest item retained by the sketch + * corresponding to a zero rank or zero probability, and ends with the first split point

+ * + *

The last (m+1)th interval starts with the last split point and ends after the last + * item retained by the sketch corresponding to a rank or probability of 1.0.

+ * + *

The sum of the probability masses of all (m+1) intervals is 1.0.

+ * + *

If the search criterion is:

+ * + *
    + *
  • INCLUSIVE, and the upper split point of an interval equals an item retained by the sketch, the interval + * will include that item. If the lower split point equals an item retained by the sketch, the interval will exclude + * that item.
    • + *
  • EXCLUSIVE, and the upper split point of an interval equals an item retained by the sketch, the interval + * will exclude that item. If the lower split point equals an item retained by the sketch, the interval will include + * that item.
    • + *
+ * + *

It is not recommended to include either the minimum or maximum items of the input stream.

+ * + * @param searchCrit the desired search criteria. + * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. + * @throws IllegalArgumentException if sketch is empty. + */ + double[] getPMF(long[] splitPoints, QuantileSearchCriteria searchCrit); + + /** + * This is equivalent to {@link #getQuantile(double, QuantileSearchCriteria) getQuantile(rank, INCLUSIVE)} + * @param rank the given normalized rank, a double in the range [0.0, 1.0]. + * @return the approximate quantile given the normalized rank. + * @throws IllegalArgumentException if sketch is empty. + */ + default long getQuantile(double rank) { + return getQuantile(rank, INCLUSIVE); + } + + /** + * Gets the approximate quantile of the given normalized rank and the given search criterion. + * + * @param rank the given normalized rank, a double in the range [0.0, 1.0]. + * @param searchCrit If INCLUSIVE, the given rank includes all quantiles ≤ + * the quantile directly corresponding to the given rank. + * If EXCLUSIVE, he given rank includes all quantiles < + * the quantile directly corresponding to the given rank. + * @return the approximate quantile given the normalized rank. + * @throws IllegalArgumentException if sketch is empty. + * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria + */ + long getQuantile(double rank, QuantileSearchCriteria searchCrit); + + /** + * Gets the lower bound of the quantile confidence interval in which the quantile of the + * given rank exists. + * + *

Although it is possible to estimate the probability that the true quantile + * exists within the quantile confidence interval specified by the upper and lower quantile bounds, + * it is not possible to guarantee the width of the quantile confidence interval + * as an additive or multiplicative percent of the true quantile.

+ * + * @param rank the given normalized rank + * @return the lower bound of the quantile confidence interval in which the quantile of the + * given rank exists. + * @throws IllegalArgumentException if sketch is empty. + */ + long getQuantileLowerBound(double rank); + + /** + * Gets the upper bound of the quantile confidence interval in which the true quantile of the + * given rank exists. + * + *

Although it is possible to estimate the probability that the true quantile + * exists within the quantile confidence interval specified by the upper and lower quantile bounds, + * it is not possible to guarantee the width of the quantile interval + * as an additive or multiplicative percent of the true quantile.

+ * + * @param rank the given normalized rank + * @return the upper bound of the quantile confidence interval in which the true quantile of the + * given rank exists. + * @throws IllegalArgumentException if sketch is empty. + */ + long getQuantileUpperBound(double rank); + + /** + * This is equivalent to {@link #getQuantiles(double[], QuantileSearchCriteria) getQuantiles(ranks, INCLUSIVE)} + * @param ranks the given array of normalized ranks, each of which must be + * in the interval [0.0,1.0]. + * @return an array of quantiles corresponding to the given array of normalized ranks. + * @throws IllegalArgumentException if sketch is empty. + */ + default long[] getQuantiles(double[] ranks) { + return getQuantiles(ranks, INCLUSIVE); + } + + /** + * Gets an array of quantiles from the given array of normalized ranks. + * + * @param ranks the given array of normalized ranks, each of which must be + * in the interval [0.0,1.0]. + * @param searchCrit if INCLUSIVE, the given ranks include all quantiles ≤ + * the quantile directly corresponding to each rank. + * @return an array of quantiles corresponding to the given array of normalized ranks. + * @throws IllegalArgumentException if sketch is empty. + * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria + */ + long[] getQuantiles(double[] ranks, QuantileSearchCriteria searchCrit); + + /** + * This is equivalent to {@link #getRank(long, QuantileSearchCriteria) getRank(quantile, INCLUSIVE)} + * @param quantile the given quantile + * @return the normalized rank corresponding to the given quantile + * @throws IllegalArgumentException if sketch is empty. + */ + default double getRank(long quantile) { + return getRank(quantile, INCLUSIVE); + } + + /** + * Gets the normalized rank corresponding to the given a quantile. + * + * @param quantile the given quantile + * @param searchCrit if INCLUSIVE the given quantile is included into the rank. + * @return the normalized rank corresponding to the given quantile + * @throws IllegalArgumentException if sketch is empty. + * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria + */ + double getRank(long quantile, QuantileSearchCriteria searchCrit); + + /** + * This is equivalent to {@link #getRanks(long[], QuantileSearchCriteria) getRanks(quantiles, INCLUSIVE)} + * @param quantiles the given array of quantiles + * @return an array of normalized ranks corresponding to the given array of quantiles. + * @throws IllegalArgumentException if sketch is empty. + */ + default double[] getRanks(long[] quantiles) { + return getRanks(quantiles, INCLUSIVE); + } + + /** + * Gets an array of normalized ranks corresponding to the given array of quantiles and the given + * search criterion. + * + * @param quantiles the given array of quantiles + * @param searchCrit if INCLUSIVE, the given quantiles include the rank directly corresponding to each quantile. + * @return an array of normalized ranks corresponding to the given array of quantiles. + * @throws IllegalArgumentException if sketch is empty. + * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria + */ + double[] getRanks(long[] quantiles, QuantileSearchCriteria searchCrit); + + /** + * Returns the current number of bytes this Sketch would require if serialized. + * @return the number of bytes this sketch would require if serialized. + */ + int getSerializedSizeBytes(); + + /** + * Gets the sorted view of this sketch + * @return the sorted view of this sketch + */ + LongsSortedView getSortedView(); + + /** + * Gets the iterator for this sketch, which is not sorted. + * @return the iterator for this sketch + */ + QuantilesLongsSketchIterator iterator(); + + /** + * Returns a byte array representation of this sketch. + * @return a byte array representation of this sketch. + */ + byte[] toByteArray(); + + /** + * Updates this sketch with the given item. + * @param item from a stream of items. NaNs are ignored. + */ + void update(long item); + +} diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsSketchIterator.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsSketchIterator.java new file mode 100644 index 000000000..7ed0d9805 --- /dev/null +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsSketchIterator.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantilescommon; + +/** + * The quantiles sketch iterator for primitive type long. + * @see QuantilesSketchIterator + * @author Zac Blanco + */ +public interface QuantilesLongsSketchIterator extends QuantilesSketchIterator { + + /** + * Gets the long quantile at the current index. + * + *

Don't call this before calling next() for the first time + * or after getting false from next().

+ * + * @return the long quantile at the current index. + */ + long getQuantile(); + +} + From c0479b299594834325731c94bc27cdc6ad0bd12f Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 9 Oct 2024 15:01:00 -0700 Subject: [PATCH 17/88] fix findbugs, includeMinMax, QuantilesUtil. --- .../quantilescommon/IncludeMinMax.java | 65 +++++++++++++++++++ .../quantilescommon/QuantilesUtil.java | 15 +++++ tools/FindBugsExcludeFilter.xml | 1 + 3 files changed, 81 insertions(+) diff --git a/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java b/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java index 203e338d2..1edfc2054 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java @@ -63,6 +63,24 @@ public FloatsPair(final float[] quantiles, final long[] cumWeights) { } } + /** A simple structure to hold a pair of arrays */ + public static class LongsPair { + /** the array of quantiles */ + public long[] quantiles; + /** the array of associated cumulative weights */ + public long[] cumWeights; + + /** + * Constructor. + * @param quantiles the array of quantiles + * @param cumWeights the array of associated cumulative weights + */ + public LongsPair(final long[] quantiles, final long[] cumWeights) { + this.quantiles = quantiles; + this.cumWeights = cumWeights; + } + } + /** * A simple structure to hold a pair of arrays * @param the item class type @@ -131,6 +149,53 @@ public static DoublesPair includeDoublesMinMax( return new DoublesPair(adjQuantiles, adjCumWeights); } + /** + * The logic to include the min and max of type double. + * @param quantilesIn The array of quantiles + * @param cumWeightsIn The array of associated cumulative weights + * @param maxItem the maximum item of the stream + * @param minItem the minimum item of the stream + * @return a DoublesPair + */ + public static LongsPair includeLongsMinMax( + final long[] quantilesIn, + final long[] cumWeightsIn, + final long maxItem, + final long minItem) { + final int lenIn = cumWeightsIn.length; + final boolean adjLow = quantilesIn[0] != minItem; //if true, adjust the low end + final boolean adjHigh = quantilesIn[lenIn - 1] != maxItem; //if true, adjust the high end + int adjLen = lenIn; //this will be the length of the local copies of quantiles and cumWeights + adjLen += adjLow ? 1 : 0; + adjLen += adjHigh ? 1 : 0; + final long[] adjQuantiles; + final long[] adjCumWeights; + if (adjLen > lenIn) { //is any adjustment required at all? + adjQuantiles = new long[adjLen]; + adjCumWeights = new long[adjLen]; + final int offset = adjLow ? 1 : 0; + System.arraycopy(quantilesIn, 0, adjQuantiles, offset, lenIn); + System.arraycopy(cumWeightsIn,0, adjCumWeights, offset, lenIn); + + //Adjust the low end if required. Don't need to adjust weight of next one because it is cumulative. + if (adjLow) { + adjQuantiles[0] = minItem; + adjCumWeights[0] = 1; + } + + if (adjHigh) { + adjQuantiles[adjLen - 1] = maxItem; + adjCumWeights[adjLen - 1] = cumWeightsIn[lenIn - 1]; + adjCumWeights[adjLen - 2] = cumWeightsIn[lenIn - 1] - 1; + } + } else { //both min and max are already in place, no adjustments are required. + adjQuantiles = quantilesIn; + adjCumWeights = cumWeightsIn; + + } //END of Adjust End Points + return new LongsPair(adjQuantiles, adjCumWeights); + } + /** * The logic to include the min and max of type float. * @param quantilesIn The array of quantiles diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesUtil.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesUtil.java index 75798c20f..529fd386d 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesUtil.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesUtil.java @@ -67,6 +67,21 @@ public static final void checkDoublesSplitPointsOrder(final double[] values) { } } + /** + * Checks the sequential validity of the given array of double values. + * They must be unique and monotonically increasing. + * @param values the given array of double values + */ + public static void checkLongsSplitPointsOrder(final long[] values) { + Objects.requireNonNull(values); + final int len = values.length; + for (int j = 0; j < len - 1; j++) { + if (values[j] < values[j + 1]) { continue; } + throw new SketchesArgumentException( + "Values must be unique and monotonically increasing."); + } + } + /** * Checks the sequential validity of the given array of float values. * They must be unique, monotonically increasing and not NaN. diff --git a/tools/FindBugsExcludeFilter.xml b/tools/FindBugsExcludeFilter.xml index 4f62a5cd6..d47e78f09 100644 --- a/tools/FindBugsExcludeFilter.xml +++ b/tools/FindBugsExcludeFilter.xml @@ -41,6 +41,7 @@ under the License. + From 7ee5cf3b0d4a74d8fe13b60f23f3797d82f0df78 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 9 Oct 2024 16:43:31 -0700 Subject: [PATCH 18/88] More changes for 17 --- pom.xml | 74 ++++++++++--------- .../apache/datasketches/kll/KllHelper.java | 25 +++++++ .../datasketches/kll/KllPreambleUtil.java | 29 +++++++- 3 files changed, 92 insertions(+), 36 deletions(-) diff --git a/pom.xml b/pom.xml index 949152680..d6635e6e4 100644 --- a/pom.xml +++ b/pom.xml @@ -18,14 +18,22 @@ specific language governing permissions and limitations under the License. --> - + 4.0.0 + + org.apache + apache + 32 + + org.apache.datasketches datasketches-java - 6.1.0-SNAPSHOT + 7.0.0 jar ${project.artifactId} @@ -58,8 +66,8 @@ under the License. - - + GitHub https://github.com/apache/${project.artifactId}/issues @@ -75,10 +83,10 @@ under the License. - 2.2.0 + 4.0.0 - 7.10.0 + 7.10.2 generate_java_files check_cpp_files @@ -96,32 +104,31 @@ under the License. ${charset.encoding} yyyy-MM-dd'T'HH-mm-ss'Z' - - 3.7.1 - 3.13.0 - 3.1.1 - 3.4.1 - 3.2.3 - 3.4.0 - 3.7.0 - 3.0.1 - 3.2.0 - 3.3.1 - 3.2.5 - 3.1.2 - - 0.16.1 - - 0.8.12 - - - 4.3.0 - - 2.16.2 - - - 1.0.0 - 4.9.10 + + 3.7.1 + 3.13.0 + 3.1.3 + 3.5.0 + 3.2.5 + 3.4.2 + 3.8.0 + 3.1.1 + 3.2.0 + 3.3.1 + 3.4.0 + + 4.9.10 + + 0.16.1 + + + 4.3.0 + + 0.8.12 + + 2.17.1 + + 1.0.0 @@ -130,6 +137,7 @@ under the License. datasketches-memory ${datasketches-memory.version} + org.testng diff --git a/src/main/java/org/apache/datasketches/kll/KllHelper.java b/src/main/java/org/apache/datasketches/kll/KllHelper.java index d60220948..21188255c 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllHelper.java @@ -42,6 +42,7 @@ import static org.apache.datasketches.kll.KllSketch.SketchType.DOUBLES_SKETCH; import static org.apache.datasketches.kll.KllSketch.SketchType.FLOATS_SKETCH; import static org.apache.datasketches.kll.KllSketch.SketchType.ITEMS_SKETCH; +import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH; import static org.apache.datasketches.quantilescommon.QuantilesAPI.UNSUPPORTED_MSG; import java.nio.ByteOrder; @@ -591,6 +592,11 @@ static void addEmptyTopLevelToCompletelyFullSketch(final KllSketch sketch) { float minFloat = Float.NaN; float maxFloat = Float.NaN; + long[] myCurLongItemsArr = null; + long[] myNewLongItemsArr = null; + long minLong = Long.MAX_VALUE; + long maxLong = Long.MIN_VALUE; + Object[] myCurItemsArr = null; Object[] myNewItemsArr = null; Object minItem = null; @@ -611,6 +617,14 @@ else if (sketchType == FLOATS_SKETCH) { maxFloat = fltSk.getMaxItem(); //assert we are following a certain growth scheme assert myCurFloatItemsArr.length == myCurTotalItemsCapacity; + } + else if (sketchType == LONGS_SKETCH) { + final KllLongsSketch lngSk = (KllLongsSketch) sketch; + myCurLongItemsArr = lngSk.getLongItemsArray(); + minLong = lngSk.getMinItem(); + maxLong = lngSk.getMaxItem(); + //assert we are following a certain growth scheme + assert myCurLongItemsArr.length == myCurTotalItemsCapacity; } else { //sketchType == ITEMS_SKETCH final KllItemsSketch itmSk = (KllItemsSketch) sketch; @@ -654,6 +668,11 @@ else if (sketchType == FLOATS_SKETCH) { myNewFloatItemsArr = new float[myNewTotalItemsCapacity]; // copy and shift the current items data into the new array System.arraycopy(myCurFloatItemsArr, 0, myNewFloatItemsArr, deltaItemsCap, myCurTotalItemsCapacity); + } + else if (sketchType == LONGS_SKETCH) { + myNewLongItemsArr = new long[myNewTotalItemsCapacity]; + // copy and shift the current items data into the new array + System.arraycopy(myCurLongItemsArr, 0, myNewLongItemsArr, deltaItemsCap, myCurTotalItemsCapacity); } else { //sketchType == ITEMS_SKETCH myNewItemsArr = new Object[myNewTotalItemsCapacity]; @@ -681,6 +700,12 @@ else if (sketchType == FLOATS_SKETCH) { fltSk.setMinItem(minFloat); fltSk.setMaxItem(maxFloat); fltSk.setFloatItemsArray(myNewFloatItemsArr); + } + else if (sketchType == LONGS_SKETCH) { + final KllLongsSketch lngSk = (KllLongsSketch) sketch; + lngSk.setMinItem(minLong); + lngSk.setMaxItem(maxLong); + lngSk.setLongItemsArray(myNewLongItemsArr); } else { //sketchType == ITEMS_SKETCH final KllItemsSketch itmSk = (KllItemsSketch) sketch; diff --git a/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java b/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java index 18e4de3f0..74a8009ea 100644 --- a/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java +++ b/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java @@ -28,6 +28,7 @@ import static org.apache.datasketches.kll.KllSketch.SketchType.DOUBLES_SKETCH; import static org.apache.datasketches.kll.KllSketch.SketchType.FLOATS_SKETCH; import static org.apache.datasketches.kll.KllSketch.SketchType.ITEMS_SKETCH; +import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH; import java.util.Objects; @@ -210,7 +211,7 @@ static String toString(final Memory mem, final SketchType sketchType, final bool * Used primarily in testing. * * @param mem the given Memory - * @param sketchType the sketch type: FLOATS_SKETCH, DOUBLES_SKETCH, or ITEMS_SKETCH. + * @param sketchType the sketch type: FLOATS_SKETCH, DOUBLES_SKETCH, LONGS_SKETCH, or ITEMS_SKETCH. * @param includeData if true, includes detail of retained data. * @param serDe must be supplied for KllItemsSketch, otherwise can be null. * @return the summary string. @@ -229,7 +230,7 @@ static String toString(final Memory mem, final SketchType sketchType, final //final boolean updatable = mySketchStructure == UPDATABLE; final boolean emptyFlag = memVal.emptyFlag; final int sketchBytes = memVal.sketchBytes; - final int typeBytes = sketchType == DOUBLES_SKETCH ? Double.BYTES : Float.BYTES; + final int typeBytes = sketchType.getBytes(); final int familyID = getMemoryFamilyID(mem); final String famName = idToFamily(familyID).toString(); @@ -295,6 +296,11 @@ static String toString(final Memory mem, final SketchType sketchType, final offsetBytes += typeBytes; sb.append(mem.getFloat(offsetBytes)).append(LS); offsetBytes += typeBytes; + } else if (sketchType == LONGS_SKETCH) { + sb.append(mem.getLong(offsetBytes)).append(LS); + offsetBytes += typeBytes; + sb.append(mem.getLong(offsetBytes)).append(LS); + offsetBytes += typeBytes; } else { //ITEMS_SKETCH sb.append("<<>>").append(LS); } @@ -311,6 +317,11 @@ static String toString(final Memory mem, final SketchType sketchType, final sb.append(mem.getFloat(offsetBytes)).append(LS); offsetBytes += typeBytes; } + } else if (sketchType == LONGS_SKETCH) { + for (int i = 0; i < itemsSpace; i++) { + sb.append(mem.getLong(offsetBytes)).append(LS); + offsetBytes += typeBytes; + } } else { //ITEMS_SKETCH sb.append("<<>>").append(LS); } @@ -338,6 +349,11 @@ static String toString(final Memory mem, final SketchType sketchType, final offsetBytes += typeBytes; sb.append(mem.getFloat(offsetBytes)).append(LS); offsetBytes += typeBytes; + } else if (sketchType == LONGS_SKETCH) { + sb.append(mem.getLong(offsetBytes)).append(LS); + offsetBytes += typeBytes; + sb.append(mem.getLong(offsetBytes)).append(LS); + offsetBytes += typeBytes; } else { //ITEMS_SKETCH sb.append(serDe.deserializeFromMemory(mem, offsetBytes, 1)[0]).append(LS); offsetBytes += serDe.sizeOf(mem, offsetBytes, 1); @@ -346,7 +362,7 @@ static String toString(final Memory mem, final SketchType sketchType, final } sb.append("RETAINED DATA").append(LS); - final int itemSpace = (sketchBytes - offsetBytes) / typeBytes; + final int itemSpace = (sketchBytes - offsetBytes) / (typeBytes == 0 ? 1 : typeBytes); if (sketchType == DOUBLES_SKETCH) { for (int i = 0; i < itemSpace; i++) { sb.append(i + ", " + mem.getDouble(offsetBytes)).append(LS); @@ -357,6 +373,11 @@ static String toString(final Memory mem, final SketchType sketchType, final sb.append(i + ", " + mem.getFloat(offsetBytes)).append(LS); offsetBytes += typeBytes; } + } else if (sketchType == LONGS_SKETCH) { + for (int i = 0; i < itemSpace; i++) { + sb.append(i + ", " + mem.getLong(offsetBytes)).append(LS); + offsetBytes += typeBytes; + } } else { //ITEMS_SKETCH final T[] itemsArr = serDe.deserializeFromMemory(mem, offsetBytes, retainedItems); for (int i = 0; i < itemsArr.length; i++) { @@ -372,6 +393,8 @@ static String toString(final Memory mem, final SketchType sketchType, final sb.append(mem.getDouble(DATA_START_ADR_SINGLE_ITEM)).append(LS); } else if (sketchType == FLOATS_SKETCH) { sb.append(mem.getFloat(DATA_START_ADR_SINGLE_ITEM)).append(LS); + } else if (sketchType == LONGS_SKETCH) { + sb.append(mem.getLong(DATA_START_ADR_SINGLE_ITEM)).append(LS); } else { //ITEMS_SKETCH sb.append(serDe.deserializeFromMemory(mem, DATA_START_ADR_SINGLE_ITEM, 1)[0]).append(LS); } From 2b44c6d963bb7733c784cf443d046cef1cb81cbe Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 9 Oct 2024 18:20:02 -0700 Subject: [PATCH 19/88] Added more robust checking of closing resources. --- .../hll/DirectAuxHashMapTest.java | 1 + .../DirectQuantilesMemoryRequestTest.java | 12 +++-- .../quantiles/DoublesSketchTest.java | 6 ++- .../theta/DirectQuickSelectSketchTest.java | 6 ++- .../theta/HeapifyWrapSerVer1and2Test.java | 48 +++++++++---------- .../datasketches/theta/UnionImplTest.java | 4 ++ 6 files changed, 45 insertions(+), 32 deletions(-) diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java index 98039616f..c913af378 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java @@ -90,6 +90,7 @@ public void checkGrow() { assertTrue(hllSketch.isMemory()); assertFalse(hllSketch.isOffHeap()); assertFalse(hllSketch.isSameResource(wmem)); + assertFalse(wmem.isAlive()); } @Test diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java index 6552d0ae3..c252eef3c 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java @@ -51,7 +51,7 @@ public void checkLimitedMemoryScenarios() { //Requesting application //########## Owning Implementation // This part would actually be part of the Memory owning implementation so it is faked here WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - + WritableMemory wmem2 = wmem; println("Initial mem size: " + wmem.getCapacity()); //########## Receiving Application @@ -74,6 +74,7 @@ public void checkLimitedMemoryScenarios() { //Requesting application //The actual Memory has been re-allocated several times, // so the above wmem reference is invalid. println("\nFinal mem size: " + wmem.getCapacity()); + assertFalse(wmem2.isAlive()); } @Test @@ -83,7 +84,7 @@ public void checkGrowBaseBuf() { final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - + WritableMemory wmem2 = wmem; println("Initial mem size: " + wmem.getCapacity()); final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); for (int i = 1; i <= u; i++) { @@ -92,6 +93,7 @@ public void checkGrowBaseBuf() { final int currentSpace = usk1.getCombinedBufferItemCapacity(); println("curCombBufItemCap: " + currentSpace); assertEquals(currentSpace, 2 * k); + assertFalse(wmem2.isAlive()); } @Test @@ -101,7 +103,7 @@ public void checkGrowCombBuf() { final int initBytes = ((2 * k) + 4) << 3; //just room for BB WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - + WritableMemory wmem2 = wmem; println("Initial mem size: " + wmem.getCapacity()); final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); for (int i = 1; i <= u; i++) { @@ -113,6 +115,7 @@ public void checkGrowCombBuf() { final int newSpace = usk1.getCombinedBufferItemCapacity(); println("newCombBurItemCap: " + newSpace); assertEquals(newCB.length, 3 * k); + assertFalse(wmem2.isAlive()); } @Test @@ -124,7 +127,7 @@ public void checkGrowFromWrappedEmptySketch() { final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - + WritableMemory wmem2 = wmem; origSketchMem.copyTo(0, wmem, 0, initBytes); UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem); assertTrue(wmem.isSameResource(usk2.getMemory())); @@ -141,6 +144,7 @@ public void checkGrowFromWrappedEmptySketch() { final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); assertEquals(mem2.getCapacity(), expectedSize); + assertFalse(wmem2.isAlive()); } @Test diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java index 8f4243bfe..ea928ba02 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java @@ -28,7 +28,6 @@ import java.nio.ByteOrder; import org.apache.datasketches.memory.DefaultMemoryRequestServer; -//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.quantilescommon.DoublesSortedView; import org.apache.datasketches.quantilescommon.DoublesSortedViewIterator; @@ -144,19 +143,21 @@ public void checkEmptyExceptions() { public void directSketchShouldMoveOntoHeapEventually() { WritableMemory wmem = WritableMemory.allocateDirect(1000, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmem2 = wmem; UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); Assert.assertTrue(sketch.isSameResource(wmem)); for (int i = 0; i < 1000; i++) { sketch.update(i); } Assert.assertFalse(sketch.isSameResource(wmem)); + Assert.assertFalse(wmem2.isAlive()); } @Test public void directSketchShouldMoveOntoHeapEventually2() { int i = 0; WritableMemory wmem = WritableMemory.allocateDirect(50, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - + WritableMemory wmem2 = wmem; UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); Assert.assertTrue(sketch.isSameResource(wmem)); for (; i < 1000; i++) { @@ -167,6 +168,7 @@ public void directSketchShouldMoveOntoHeapEventually2() { break; } } + Assert.assertFalse(wmem2.isAlive()); } @Test diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java index 63957c3cd..ef6004a1b 100644 --- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java @@ -876,11 +876,12 @@ public void checkMoveAndResize() { int bytes = Sketches.getMaxUpdateSketchBytes(k); WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - + WritableMemory wmem2 = wmem; UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); assertTrue(sketch.isSameResource(wmem)); for (int i = 0; i < u; i++) { sketch.update(i); } assertFalse(sketch.isSameResource(wmem)); + assertFalse(wmem2.isAlive()); } @Test @@ -889,7 +890,7 @@ public void checkReadOnlyRebuildResize() { int u = 2 * k; int bytes = Sketches.getMaxUpdateSketchBytes(k); WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - + WritableMemory wmem2 = wmem; UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); for (int i = 0; i < u; i++) { sketch.update(i); } double est1 = sketch.getEstimate(); @@ -910,6 +911,7 @@ public void checkReadOnlyRebuildResize() { } catch (SketchesReadOnlyException e) { //expected } + assertFalse(wmem2.isAlive()); } @Test diff --git a/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java b/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java index 6c90eafe4..5e81b1808 100644 --- a/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java +++ b/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java @@ -293,7 +293,7 @@ public void checkWrapCompactSketchAssumedDefaultSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertTrue(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV2 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer2(sv3csk, seed)); @@ -301,7 +301,7 @@ public void checkWrapCompactSketchAssumedDefaultSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV1 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer1(sv3csk)); @@ -309,7 +309,7 @@ public void checkWrapCompactSketchAssumedDefaultSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } } @Test @@ -329,7 +329,7 @@ public void checkWrapCompactSketchAssumedDifferentSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertTrue(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV2 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer2(sv3csk, seed)); @@ -337,7 +337,7 @@ public void checkWrapCompactSketchAssumedDifferentSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV1 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer1(sv3csk)); @@ -345,7 +345,7 @@ public void checkWrapCompactSketchAssumedDifferentSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), defaultSeedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } } @Test @@ -365,7 +365,7 @@ public void checkWrapCompactSketchGivenDefaultSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertTrue(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV2 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer2(sv3csk, seed)); @@ -373,7 +373,7 @@ public void checkWrapCompactSketchGivenDefaultSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV1 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer1(sv3csk)); @@ -381,7 +381,7 @@ public void checkWrapCompactSketchGivenDefaultSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {/* ignore */} + if (wmem.isAlive()) { wmem.close(); } } @Test @@ -401,7 +401,7 @@ public void checkWrapCompactSketchGivenDifferentSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertTrue(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV2 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer2(sv3csk, seed)); @@ -409,7 +409,7 @@ public void checkWrapCompactSketchGivenDifferentSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV1 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer1(sv3csk)); @@ -417,7 +417,7 @@ public void checkWrapCompactSketchGivenDifferentSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } } @Test @@ -437,7 +437,7 @@ public void checkWrapSketchAssumedDefaultSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertTrue(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV2 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer2(sv3csk, seed)); @@ -445,7 +445,7 @@ public void checkWrapSketchAssumedDefaultSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV1 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer1(sv3csk)); @@ -453,7 +453,7 @@ public void checkWrapSketchAssumedDefaultSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } } @Test @@ -473,7 +473,7 @@ public void checkWrapSketchAssumedDifferentSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertTrue(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV2 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer2(sv3csk, seed)); @@ -481,7 +481,7 @@ public void checkWrapSketchAssumedDifferentSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV1 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer1(sv3csk)); @@ -489,7 +489,7 @@ public void checkWrapSketchAssumedDifferentSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), defaultSeedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } } @Test @@ -509,7 +509,7 @@ public void checkWrapSketchGivenDefaultSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertTrue(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV2 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer2(sv3csk, seed)); @@ -517,7 +517,7 @@ public void checkWrapSketchGivenDefaultSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV1 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer1(sv3csk)); @@ -525,7 +525,7 @@ public void checkWrapSketchGivenDefaultSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } } @Test @@ -545,7 +545,7 @@ public void checkWrapSketchGivenDifferentSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertTrue(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV2 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer2(sv3csk, seed)); @@ -553,7 +553,7 @@ public void checkWrapSketchGivenDifferentSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } //SV1 test wmem = putOffHeap(BackwardConversions.convertSerVer3toSerVer1(sv3csk)); @@ -561,7 +561,7 @@ public void checkWrapSketchGivenDifferentSeed() { assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate()); assertEquals(sv3cskResult.getSeedHash(), seedHash); assertFalse(sv3cskResult.isDirect()); - try { wmem.close(); } catch (Exception e) {} + if (wmem.isAlive()) { wmem.close(); } } private static WritableMemory putOffHeap(Memory heapMem) { diff --git a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java index f28212061..e569175e7 100644 --- a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java +++ b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java @@ -200,10 +200,12 @@ public void checkMoveAndResize() { ResourceScope scope = ResourceScope.newConfinedScope(); final WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2, 1, scope, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + final WritableMemory wmemA = wmem; final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); assertTrue(sketch.isSameResource(wmem)); final WritableMemory wmem2 = WritableMemory.allocateDirect(bytes / 2, 1, scope, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + final WritableMemory wmemB = wmem2; final Union union = SetOperation.builder().buildUnion(wmem2); assertTrue(union.isSameResource(wmem2)); @@ -212,6 +214,8 @@ public void checkMoveAndResize() { final Union union2 = SetOperation.builder().buildUnion(); //on-heap union assertFalse(union2.isSameResource(wmem2)); //obviously not + assertFalse(wmemA.isAlive()); + assertFalse(wmemB.isAlive()); } @Test From 935099185b1aaeed35bc8da4f55e366abdd2e153 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 10 Oct 2024 11:10:57 -0700 Subject: [PATCH 20/88] remove extraneous offset increment --- src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java b/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java index 74a8009ea..e11228669 100644 --- a/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java +++ b/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java @@ -383,7 +383,6 @@ static String toString(final Memory mem, final SketchType sketchType, final for (int i = 0; i < itemsArr.length; i++) { sb.append(i + ", " + serDe.toString(itemsArr[i])).append(LS); } - offsetBytes += serDe.sizeOf(mem, offsetBytes, retainedItems); } } else if (myStructure == COMPACT_SINGLE) { From e2f6b855c1ade13980a5ce42f3beb0db22f9372e Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Sun, 13 Oct 2024 10:51:42 -0700 Subject: [PATCH 21/88] Updates to ds-java17 --- README.md | 2 - .../datasketches/common/MemoryStatus.java | 54 +++++++++++++ .../org/apache/datasketches/common/Util.java | 3 +- .../datasketches/hll/DirectHllArray.java | 4 - .../datasketches/kll/KllHeapItemsSketch.java | 2 +- .../datasketches/kll/KllPreambleUtil.java | 1 - .../quantiles/DirectUpdateDoublesSketch.java | 2 +- .../quantilescommon/DoublesSortedView.java | 5 +- .../GenericInequalitySearch.java | 2 +- .../quantilescommon/InequalitySearch.java | 2 +- .../QuantileSearchCriteria.java | 2 +- .../quantilescommon/QuantilesAPI.java | 2 +- .../datasketches/req/BaseReqSketch.java | 2 +- .../apache/datasketches/theta/AnotBimpl.java | 6 -- .../theta/ConcurrentHeapThetaBuffer.java | 4 +- .../theta/ConcurrentSharedThetaSketch.java | 15 ++-- .../theta/DirectCompactSketch.java | 6 +- .../theta/DirectQuickSelectSketch.java | 4 + .../theta/DirectQuickSelectSketchR.java | 6 +- .../theta/EmptyCompactSketch.java | 10 --- .../datasketches/theta/HeapCompactSketch.java | 10 --- .../datasketches/theta/HeapUpdateSketch.java | 10 --- .../datasketches/theta/IntersectionImpl.java | 14 +++- .../apache/datasketches/theta/Rebuilder.java | 2 +- .../datasketches/theta/SetOperation.java | 17 +--- .../theta/SetOperationBuilder.java | 2 +- .../datasketches/theta/SingleItemSketch.java | 10 --- .../org/apache/datasketches/theta/Sketch.java | 42 ++-------- .../apache/datasketches/theta/Sketches.java | 10 +-- .../apache/datasketches/theta/UnionImpl.java | 14 +++- .../BloomFilterCrossLanguageTest.java | 80 +++++++++++++++++++ .../filters/bloomfilter/BloomFilterTest.java | 2 - 32 files changed, 202 insertions(+), 145 deletions(-) create mode 100644 src/main/java/org/apache/datasketches/common/MemoryStatus.java create mode 100644 src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java diff --git a/README.md b/README.md index 3190036d1..80517d304 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,6 @@ --> [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.datasketches/datasketches-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.apache.datasketches/datasketches-java) -[![Language grade: Java](https://img.shields.io/lgtm/grade/java/g/apache/datasketches-java.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/apache/datasketches-java/context:java) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/apache/datasketches-java.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/apache/datasketches-java/alerts/) [![Coverage Status](https://coveralls.io/repos/github/apache/datasketches-java/badge.svg)](https://coveralls.io/github/apache/datasketches-java) ================= diff --git a/src/main/java/org/apache/datasketches/common/MemoryStatus.java b/src/main/java/org/apache/datasketches/common/MemoryStatus.java new file mode 100644 index 000000000..564797e32 --- /dev/null +++ b/src/main/java/org/apache/datasketches/common/MemoryStatus.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.common; + +import org.apache.datasketches.memory.Memory; + +/** + * Methods for inquiring the status of a backing Memory object. + */ +public interface MemoryStatus { + + /** + * Returns true if this object's internal data is backed by a Memory object, + * which may be on-heap or off-heap. + * @return true if this object's internal data is backed by a Memory object. + */ + default boolean hasMemory() { return false; } + + /** + * Returns true if this object's internal data is backed by direct (off-heap) Memory. + * @return true if this object's internal data is backed by direct (off-heap) Memory. + */ + default boolean isDirect() { return false; } + + /** + * Returns true if the backing resource of this is identical with the backing resource + * of that. The capacities must be the same. If this is a region, + * the region offset must also be the same. + * + * @param that A different non-null and alive Memory object. + * @return true if the backing resource of this is identical with the backing resource + * of that. + * @throws SketchesArgumentException if that is not alive (already closed). + */ + default boolean isSameResource(final Memory that) { return false; } + +} diff --git a/src/main/java/org/apache/datasketches/common/Util.java b/src/main/java/org/apache/datasketches/common/Util.java index 729b92f3f..19a8ee614 100644 --- a/src/main/java/org/apache/datasketches/common/Util.java +++ b/src/main/java/org/apache/datasketches/common/Util.java @@ -245,7 +245,8 @@ public static String zeroPad(final String s, final int fieldLength) { /** * Prepend or postpend the given string with the given character to fill the given field length. - * If the given string is equal to or greater than the given field length, it will be returned without modification. + * If the given string is equal to or greater than the given field length, it will be returned + * without modification. * @param s the given string * @param fieldLength the desired field length * @param padChar the desired pad character diff --git a/src/main/java/org/apache/datasketches/hll/DirectHllArray.java b/src/main/java/org/apache/datasketches/hll/DirectHllArray.java index 07835da69..e0c66c0aa 100644 --- a/src/main/java/org/apache/datasketches/hll/DirectHllArray.java +++ b/src/main/java/org/apache/datasketches/hll/DirectHllArray.java @@ -56,7 +56,6 @@ abstract class DirectHllArray extends AbstractHllArray { WritableMemory wmem; Memory mem; Object memObj; - //long memAdd; final boolean compact; private static int checkMemCompactFlag(final WritableMemory wmem, final int lgConfigK) { @@ -70,7 +69,6 @@ private static int checkMemCompactFlag(final WritableMemory wmem, final int lgCo this.wmem = wmem; mem = wmem; memObj = wmem.getArray(); - //memAdd = wmem.getCumulativeOffset(0L); compact = extractCompactFlag(mem); insertEmptyFlag(wmem, false); } @@ -81,7 +79,6 @@ private static int checkMemCompactFlag(final WritableMemory wmem, final int lgCo wmem = null; this.mem = mem; memObj = ((WritableMemory) mem).getArray(); - //memAdd = mem.getCumulativeOffset(0L); compact = extractCompactFlag(mem); } @@ -90,7 +87,6 @@ final void updateMemory(final WritableMemory newWmem) { wmem = newWmem; mem = newWmem; memObj = wmem.getArray(); - //memAdd = wmem.getCumulativeOffset(0L); } @Override diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java index e58516fd3..87e3a542a 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java @@ -36,7 +36,7 @@ import org.apache.datasketches.memory.WritableMemory; /** - * This class implements an on-heap doubles KllSketch. + * This class implements an on-heap items KllSketch. * *

Please refer to the documentation in the package-info:
* {@link org.apache.datasketches.kll}

diff --git a/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java b/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java index e11228669..f2c3847ae 100644 --- a/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java +++ b/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java @@ -227,7 +227,6 @@ static String toString(final Memory mem, final SketchType sketchType, final final String flagsStr = (flags) + ", 0x" + (Integer.toHexString(flags)) + ", " + zeroPad(Integer.toBinaryString(flags), 8); final int preInts = memVal.preInts; //?? - //final boolean updatable = mySketchStructure == UPDATABLE; final boolean emptyFlag = memVal.emptyFlag; final int sketchBytes = memVal.sketchBytes; final int typeBytes = sketchType.getBytes(); diff --git a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java index 0998223e8..7a7a7a850 100644 --- a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java @@ -268,7 +268,7 @@ private WritableMemory growCombinedMemBuffer(final int itemSpaceNeeded) { final WritableMemory newMem = memReqSvr.request(mem_, needBytes); mem_.copyTo(0, newMem, 0, memBytes); - if (mem_.isCloseable()) { memReqSvr.requestClose(mem_, newMem); } + memReqSvr.requestClose(mem_, newMem); return newMem; } diff --git a/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java index bdc3cc75c..98616661f 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java @@ -148,7 +148,7 @@ default double[] getPMF(double[] splitPoints, QuantileSearchCriteria searchCrit * the quantile directly corresponding to the given rank. * @return the approximate quantile given the normalized rank. * @throws IllegalArgumentException if sketch is empty. - * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria + * @see QuantileSearchCriteria */ double getQuantile(double rank, QuantileSearchCriteria searchCrit); @@ -165,7 +165,7 @@ default double[] getPMF(double[] splitPoints, QuantileSearchCriteria searchCrit * @param searchCrit if INCLUSIVE the given quantile is included into the rank. * @return the normalized rank corresponding to the given quantile. * @throws IllegalArgumentException if sketch is empty. - * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria + * @see QuantileSearchCriteria */ double getRank(double quantile, QuantileSearchCriteria searchCrit); @@ -173,4 +173,3 @@ default double[] getPMF(double[] splitPoints, QuantileSearchCriteria searchCrit DoublesSortedViewIterator iterator(); } - diff --git a/src/main/java/org/apache/datasketches/quantilescommon/GenericInequalitySearch.java b/src/main/java/org/apache/datasketches/quantilescommon/GenericInequalitySearch.java index e650fef9b..7b9d6d665 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/GenericInequalitySearch.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/GenericInequalitySearch.java @@ -41,7 +41,7 @@ *

Given a sorted array of values arr[] and a search key value v, the algorithms for * the searching criteria are given with each enum criterion.

* - * @see + * @see * Sketching Quantiles and Ranks Tutorial * @author Lee Rhodes */ diff --git a/src/main/java/org/apache/datasketches/quantilescommon/InequalitySearch.java b/src/main/java/org/apache/datasketches/quantilescommon/InequalitySearch.java index 51b013573..21af2b531 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/InequalitySearch.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/InequalitySearch.java @@ -40,7 +40,7 @@ *

Given a sorted array of values arr[] and a search key value v, the algorithms for * the searching criteria are given with each enum criterion.

* - * @see + * @see * Sketching Quantiles and Ranks Tutorial * @author Lee Rhodes */ diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantileSearchCriteria.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantileSearchCriteria.java index 8ea3c3415..e7b9e6ef6 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantileSearchCriteria.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantileSearchCriteria.java @@ -22,7 +22,7 @@ /** * These search criteria are used by the KLL, REQ and Classic Quantiles sketches in the DataSketches library. * - * @see + * @see * Sketching Quantiles and Ranks Tutorial * * @author Lee Rhodes diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java index b70843bb4..9943fefee 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java @@ -194,7 +194,7 @@ *

[*] Note that obtaining epsilon may require using a similar function but with more parameters * based on the specific sketch implementation.

* - * @see + * @see * Sketching Quantiles and Ranks, Tutorial * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria * diff --git a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java index e587cd633..2460308fc 100644 --- a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java +++ b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java @@ -28,7 +28,7 @@ * This abstract class provides a single place to define and document the public API * for the Relative Error Quantiles Sketch. * - * @see + * @see * Sketching Quantiles and Ranks Tutorial * * @author Lee Rhodes diff --git a/src/main/java/org/apache/datasketches/theta/AnotBimpl.java b/src/main/java/org/apache/datasketches/theta/AnotBimpl.java index d2161c995..e7b2c99eb 100644 --- a/src/main/java/org/apache/datasketches/theta/AnotBimpl.java +++ b/src/main/java/org/apache/datasketches/theta/AnotBimpl.java @@ -26,7 +26,6 @@ import java.util.Arrays; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.thetacommon.ThetaUtil; @@ -148,11 +147,6 @@ int getRetainedEntries() { return curCount_; } - @Override - public boolean isSameResource(final Memory that) { - return false; - } - //restricted private static long[] getHashArrA(final Sketch skA) { //returns a new array diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java index 741db2f72..e7b3ddaac 100644 --- a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java +++ b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java @@ -94,7 +94,7 @@ private static int computeLogBufferSize(final int lgNomLongs, final long exactSi * @param hash to be propagated */ private boolean propagateToSharedSketch(final long hash) { - //noinspection StatementWithEmptyBody + //no inspection StatementWithEmptyBody while (localPropagationInProgress.get()) { } //busy wait until previous propagation completed localPropagationInProgress.set(true); @@ -108,7 +108,7 @@ private boolean propagateToSharedSketch(final long hash) { * Propagates the content of the buffer as a sketch to the shared sketch */ private void propagateToSharedSketch() { - //noinspection StatementWithEmptyBody + //no inspection StatementWithEmptyBody while (localPropagationInProgress.get()) { } //busy wait until previous propagation completed diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentSharedThetaSketch.java b/src/main/java/org/apache/datasketches/theta/ConcurrentSharedThetaSketch.java index 49a8140c3..cdc843f8b 100644 --- a/src/main/java/org/apache/datasketches/theta/ConcurrentSharedThetaSketch.java +++ b/src/main/java/org/apache/datasketches/theta/ConcurrentSharedThetaSketch.java @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.datasketches.common.MemoryStatus; import org.apache.datasketches.memory.WritableMemory; /** @@ -30,7 +31,7 @@ * * @author eshcar */ -interface ConcurrentSharedThetaSketch { +interface ConcurrentSharedThetaSketch extends MemoryStatus { long NOT_SINGLE_HASH = -1L; double MIN_ERROR = 0.0000001; @@ -63,8 +64,8 @@ static long computeExactLimit(long k, double error) { void endPropagation(AtomicBoolean localPropagationInProgress, boolean isEager); /** - * Returns the value of the volatile theta manged by the shared sketch - * @return the value of the volatile theta manged by the shared sketch + * Returns the value of the volatile theta managed by the shared sketch + * @return the value of the volatile theta managed by the shared sketch */ long getVolatileTheta(); @@ -124,10 +125,10 @@ boolean propagate(final AtomicBoolean localPropagationInProgress, final Sketch s // //For the external user all of the below methods can be obtained by casting the shared //sketch to UpdateSketch. However, these methods here also act as an alias so that an - //attempt to access these methods from the local buffer will be divered to the shared + //attempt to access these methods from the local buffer will be diverted to the shared //sketch. - //From Sketch + //From Sketch and MemoryStatus int getCompactBytes(); @@ -139,10 +140,6 @@ boolean propagate(final AtomicBoolean localPropagationInProgress, final Sketch s double getUpperBound(int numStdDev); - boolean hasMemory(); - - boolean isDirect(); - boolean isEmpty(); boolean isEstimationMode(); diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java index de0e6e43b..0f69ec3c2 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java @@ -109,12 +109,12 @@ public long getThetaLong() { @Override public boolean hasMemory() { - return true; + return mem_ != null; } @Override public boolean isDirect() { - return mem_.isDirect(); + return hasMemory() ? mem_.isDirect() : false; } @Override @@ -132,7 +132,7 @@ public boolean isOrdered() { @Override public boolean isSameResource(final Memory that) { - return mem_.isSameResource(that); + return hasMemory() ? mem_.isSameResource(that) : false; } @Override diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java index a1ac53c6d..ad9051a08 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java @@ -323,6 +323,10 @@ UpdateReturnState hashUpdate(final long hash) { memReqSvr_ = (memReqSvr_ == null) ? wmem_.getMemoryRequestServer() : memReqSvr_; + if (memReqSvr_ == null) { //in case the MRS is not enabled or null. + throw new SketchesArgumentException("Out of Memory, MemoryRequestServer is null, cannot expand."); + } + final WritableMemory newDstMem = memReqSvr_.request(wmem_,reqBytes); moveAndResize(wmem_, preambleLongs, lgArrLongs, newDstMem, tgtLgArrLongs, thetaLong); diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java index a3ffebc14..fb2aed2a5 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java +++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java @@ -144,12 +144,12 @@ public long getThetaLong() { @Override public boolean hasMemory() { - return true; + return wmem_ != null; } @Override public boolean isDirect() { - return wmem_.isDirect(); + return hasMemory() ? wmem_.isDirect() : false; } @Override @@ -159,7 +159,7 @@ public boolean isEmpty() { @Override public boolean isSameResource(final Memory that) { - return wmem_.isSameResource(that); + return hasMemory() ? wmem_.isSameResource(that) : false; } @Override diff --git a/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java b/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java index ae481a425..8f6e4972a 100644 --- a/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java @@ -91,16 +91,6 @@ public long getThetaLong() { return Long.MAX_VALUE; } - @Override - public boolean hasMemory() { - return false; - } - - @Override - public boolean isDirect() { - return false; - } - @Override public boolean isEmpty() { return true; diff --git a/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java b/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java index 479aa3eeb..f394e9303 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java @@ -102,16 +102,6 @@ public long getThetaLong() { return thetaLong_; } - @Override - public boolean hasMemory() { - return false; - } - - @Override - public boolean isDirect() { - return false; - } - @Override public boolean isEmpty() { return empty_; diff --git a/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java b/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java index 1cc6d75cd..49734a9e8 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java @@ -66,16 +66,6 @@ public int getCurrentBytes() { return (preLongs + dataLongs) << 3; } - @Override - public boolean isDirect() { - return false; - } - - @Override - public boolean hasMemory() { - return false; - } - //UpdateSketch @Override diff --git a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java index 509ec2f93..fc81d1124 100644 --- a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java +++ b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java @@ -336,14 +336,24 @@ public CompactSketch getResult(final boolean dstOrdered, final WritableMemory ds dstMem, compactCache); } + @Override + public boolean hasMemory() { + return wmem_ != null; + } + @Override public boolean hasResult() { - return wmem_ != null ? wmem_.getInt(RETAINED_ENTRIES_INT) >= 0 : curCount_ >= 0; + return hasMemory() ? wmem_.getInt(RETAINED_ENTRIES_INT) >= 0 : curCount_ >= 0; + } + + @Override + public boolean isDirect() { + return hasMemory() ? wmem_.isDirect() : false; } @Override public boolean isSameResource(final Memory that) { - return wmem_ != null ? wmem_.isSameResource(that) : false; + return hasMemory() ? wmem_.isSameResource(that) : false; } @Override diff --git a/src/main/java/org/apache/datasketches/theta/Rebuilder.java b/src/main/java/org/apache/datasketches/theta/Rebuilder.java index 07093f652..b6e3de342 100644 --- a/src/main/java/org/apache/datasketches/theta/Rebuilder.java +++ b/src/main/java/org/apache/datasketches/theta/Rebuilder.java @@ -84,7 +84,7 @@ static final void quickSelectAndRebuild(final WritableMemory mem, final int prea } /** - * Moves me (the entire sketch) to a new larger Memory location and rebuilds the hash table. + * Moves me (the entire updatable sketch) to a new larger Memory location and rebuilds the hash table. * This assumes a Memory preamble of standard form with the correct value of thetaLong. * Afterwards, the caller must update the local Memory reference, lgArrLongs * and hashTableThreshold from the dstMemory and free the source Memory. diff --git a/src/main/java/org/apache/datasketches/theta/SetOperation.java b/src/main/java/org/apache/datasketches/theta/SetOperation.java index 4d8ebf0c4..b89ca9703 100644 --- a/src/main/java/org/apache/datasketches/theta/SetOperation.java +++ b/src/main/java/org/apache/datasketches/theta/SetOperation.java @@ -25,6 +25,7 @@ import static org.apache.datasketches.theta.PreambleUtil.SER_VER_BYTE; import org.apache.datasketches.common.Family; +import org.apache.datasketches.common.MemoryStatus; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; @@ -35,7 +36,7 @@ * * @author Lee Rhodes */ -public abstract class SetOperation { +public abstract class SetOperation implements MemoryStatus { static final int CONST_PREAMBLE_LONGS = 3; SetOperation() {} @@ -237,20 +238,6 @@ public static int getMaxAnotBResultBytes(final int nomEntries) { */ public abstract Family getFamily(); - /** - * Returns true if the backing resource of this is identical with the backing resource - * of that. The capacities must be the same. If this is a region, - * the region offset must also be the same. - * - *

Note: Only certain set operators during stateful operations can be serialized. - * Only when they are stored into Memory will this be relevant.

- * - * @param that A different non-null object - * @return true if the backing resource of this is the same as the backing resource - * of that. - */ - public abstract boolean isSameResource(Memory that); - //restricted /** diff --git a/src/main/java/org/apache/datasketches/theta/SetOperationBuilder.java b/src/main/java/org/apache/datasketches/theta/SetOperationBuilder.java index d788418dc..4a35cf67d 100644 --- a/src/main/java/org/apache/datasketches/theta/SetOperationBuilder.java +++ b/src/main/java/org/apache/datasketches/theta/SetOperationBuilder.java @@ -66,7 +66,7 @@ public SetOperationBuilder() { * Sets the Maximum Nominal Entries (max K) for this set operation. The effective value of K of the result of a * Set Operation can be less than max K, but never greater. * The minimum value is 16 and the maximum value is 67,108,864, which is 2^26. - * @param nomEntries Nominal Entres + * @param nomEntries Nominal Entries * This will become the ceiling power of 2 if it is not a power of 2. * @return this SetOperationBuilder */ diff --git a/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java b/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java index a4bac21c9..3cfc13b1e 100644 --- a/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java +++ b/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java @@ -343,16 +343,6 @@ public double getUpperBound(final int numStdDev) { return 1.0; } - @Override - public boolean hasMemory() { - return false; - } - - @Override - public boolean isDirect() { - return false; - } - @Override public boolean isEmpty() { return false; diff --git a/src/main/java/org/apache/datasketches/theta/Sketch.java b/src/main/java/org/apache/datasketches/theta/Sketch.java index d13fe9b80..89618bc23 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketch.java +++ b/src/main/java/org/apache/datasketches/theta/Sketch.java @@ -32,6 +32,7 @@ import static org.apache.datasketches.thetacommon.HashOperations.count; import org.apache.datasketches.common.Family; +import org.apache.datasketches.common.MemoryStatus; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; @@ -44,7 +45,7 @@ * * @author Lee Rhodes */ -public abstract class Sketch { +public abstract class Sketch implements MemoryStatus { static final int DEFAULT_LG_RESIZE_FACTOR = 3; //Unique to Heap Sketch() {} @@ -292,14 +293,11 @@ public double getLowerBound(final int numStdDev) { /** * Returns the maximum number of storage bytes required for a CompactSketch with the given - * number of actual entries. Note that this assumes the worse case of the sketch in - * estimation mode, which requires storing theta and count. - * @param numberOfEntries the actual number of entries stored with the CompactSketch. + * number of actual entries. + * @param numberOfEntries the actual number of retained entries stored in the sketch. * @return the maximum number of storage bytes required for a CompactSketch with the given number - * of entries. - * @deprecated as a public method. Use {@link #getCompactSketchMaxBytes(int) instead} + * of retained entries. */ - @Deprecated public static int getMaxCompactSketchBytes(final int numberOfEntries) { if (numberOfEntries == 0) { return 8; } if (numberOfEntries == 1) { return 16; } @@ -386,26 +384,12 @@ public double getUpperBound(final int numStdDev) { : getRetainedEntries(true); } - /** - * Returns true if this sketch's data structure is backed by Memory or WritableMemory. - * @return true if this sketch's data structure is backed by Memory or WritableMemory. - */ - public abstract boolean hasMemory(); - /** * Returns true if this sketch is in compact form. * @return true if this sketch is in compact form. */ public abstract boolean isCompact(); - /** - * Returns true if the this sketch's internal data structure is backed by direct (off-heap) - * Memory. - * @return true if the this sketch's internal data structure is backed by direct (off-heap) - * Memory. - */ - public abstract boolean isDirect(); - /** * See Empty * @return true if empty. @@ -427,18 +411,6 @@ public boolean isEstimationMode() { */ public abstract boolean isOrdered(); - /** - * Returns true if the backing resource of this is identical with the backing resource - * of that. The capacities must be the same. If this is a region, - * the region offset must also be the same. - * @param that A different non-null object - * @return true if the backing resource of this is the same as the backing resource - * of that. - */ - public boolean isSameResource(final Memory that) { - return false; - } - /** * Returns a HashIterator that can be used to iterate over the retained hash values of the * Theta sketch. @@ -605,8 +577,8 @@ public static String toString(final Memory mem) { abstract int getCurrentPreambleLongs(); /** - * Returns the Memory object if it exists, otherwise null. - * @return the Memory object if it exists, otherwise null. + * Returns the backing Memory object if it exists, otherwise null. + * @return the backing Memory object if it exists, otherwise null. */ abstract Memory getMemory(); diff --git a/src/main/java/org/apache/datasketches/theta/Sketches.java b/src/main/java/org/apache/datasketches/theta/Sketches.java index 077d9d5ae..2e7fa0915 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketches.java +++ b/src/main/java/org/apache/datasketches/theta/Sketches.java @@ -80,15 +80,11 @@ public static int getMaxAnotBResultBytes(final int maxNomEntries) { /** * Returns the maximum number of storage bytes required for a CompactSketch with the given - * number of actual entries. Note that this assumes the worse case of the sketch in - * estimation mode, which requires storing theta and count. - * @param numberOfEntries the actual number of entries stored with the CompactSketch. + * number of actual entries. + * @param numberOfEntries the actual number of retained entries stored in the sketch. * @return the maximum number of storage bytes required for a CompactSketch with the given number - * of entries. - * @see Sketch#getMaxCompactSketchBytes(int) - * @deprecated as a public method. Use {@link #getCompactSketchMaxBytes(int) getCompactSketchMaxBytes(int)} instead. + * of retained entries. */ - @Deprecated public static int getMaxCompactSketchBytes(final int numberOfEntries) { return Sketch.getMaxCompactSketchBytes(numberOfEntries); } diff --git a/src/main/java/org/apache/datasketches/theta/UnionImpl.java b/src/main/java/org/apache/datasketches/theta/UnionImpl.java index 3ea5ca557..be2980801 100644 --- a/src/main/java/org/apache/datasketches/theta/UnionImpl.java +++ b/src/main/java/org/apache/datasketches/theta/UnionImpl.java @@ -262,10 +262,22 @@ public CompactSketch getResult(final boolean dstOrdered, final WritableMemory ds minThetaLong, curCountOut, seedHash, empty, true, dstOrdered, dstOrdered, dstMem, compactCacheOut); } + @Override + public boolean hasMemory() { + return gadget_ instanceof DirectQuickSelectSketchR + ? gadget_.hasMemory() : false; + } + + @Override + public boolean isDirect() { + return gadget_ instanceof DirectQuickSelectSketchR + ? gadget_.isDirect() : false; + } + @Override public boolean isSameResource(final Memory that) { return gadget_ instanceof DirectQuickSelectSketchR - ? gadget_.getMemory().isSameResource(that) : false; + ? gadget_.isSameResource(that) : false; } @Override diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java new file mode 100644 index 000000000..12c6f20df --- /dev/null +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.filters.bloomfilter; + +import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; +import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.TestUtil.cppPath; +import static org.apache.datasketches.common.TestUtil.javaPath; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; + +import org.apache.datasketches.memory.Memory; +import org.testng.annotations.Test; + +/** + * Serialize binary sketches to be tested by C++ code. + * Test deserialization of binary sketches serialized by C++ code. + */ +public class BloomFilterCrossLanguageTest { + + @Test(groups = {GENERATE_JAVA_FILES}) + public void generatBloomFilterBinariesForCompatibilityTesting() throws IOException { + final int[] nArr = {0, 10_000, 2_000_000, 300_000_00}; + final short[] hArr = {3, 5}; + for (int n : nArr) { + for (short numHashes : hArr) { + final long configBits = Math.max(n, 1000L); // so empty still has valid bit size + BloomFilter bf = BloomFilterBuilder.createBySize(configBits, numHashes); + for (int i = 0; i < n / 10; ++i) { + bf.update(i); + } + if (n > 0) { bf.update(Float.NaN); } + assertEquals(bf.isEmpty(), n == 0); + assertTrue(bf.isEmpty() || (bf.getBitsUsed() > (n / 10))); + Files.newOutputStream(javaPath.resolve("bf_n" + n + "_h" + numHashes + "_java.sk")).write(bf.toByteArray()); + } + } + } + + @Test(groups = {CHECK_CPP_FILES}) + public void readBloomFilterBinariesForCompatibilityTesting() throws IOException { + final int[] nArr = {0, 10_000, 2_000_000, 300_000_00}; + final short[] hArr = {3, 5}; + for (int n : nArr) { + for (short numHashes : hArr) { + final byte[] bytes = Files.readAllBytes(cppPath.resolve("bf_n" + n + "_h" + numHashes + "_cpp.sk")); + final BloomFilter bf = BloomFilter.heapify(Memory.wrap(bytes)); + assertEquals(bf.isEmpty(), n == 0); + assertTrue(bf.isEmpty() || (bf.getBitsUsed() > (n / 10))); + + for (int i = 0; i < n / 10; ++i) { + assertTrue(bf.query(i)); + } + if (n > 0) { + assert(bf.query(Double.NaN)); + } + } + } + } +} diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java index 81f9e0d2e..25bef3643 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java @@ -31,7 +31,6 @@ import org.apache.datasketches.common.SketchesReadOnlyException; import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; -//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; @@ -58,7 +57,6 @@ public void createNewFilterTest() throws Exception { WritableMemory wmem; try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(sizeBytes)).scope()) { - final BloomFilter bf2 = new BloomFilter(numBits, numHashes, seed, wmem); assertTrue(bf2.isEmpty()); assertTrue(bf2.hasMemory()); From 9f4813885806521a997ef7464b60f8ed14a7b7f5 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Sun, 13 Oct 2024 10:53:31 -0700 Subject: [PATCH 22/88] remove unnecessary cast --- src/main/java/org/apache/datasketches/theta/UnionImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/theta/UnionImpl.java b/src/main/java/org/apache/datasketches/theta/UnionImpl.java index be2980801..bac05de74 100644 --- a/src/main/java/org/apache/datasketches/theta/UnionImpl.java +++ b/src/main/java/org/apache/datasketches/theta/UnionImpl.java @@ -333,7 +333,7 @@ public void union(final Sketch sketchIn) { if (sketchIn.isOrdered() && (sketchIn instanceof CompactSketch)) { //Use early stop //Ordered, thus compact if (sketchIn.hasMemory()) { - final Memory skMem = ((CompactSketch) sketchIn).getMemory(); + final Memory skMem = sketchIn.getMemory(); final int preambleLongs = skMem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F; for (int i = 0; i < curCountIn; i++ ) { final int offsetBytes = preambleLongs + i << 3; From 3a5e67ce66bbb85424363bb73db9a7c57b71f74f Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 4 Nov 2024 19:01:04 -0500 Subject: [PATCH 23/88] Update gitignore, pom, and fix one warning --- .gitignore | 3 +++ pom.xml | 2 +- src/main/java/org/apache/datasketches/theta/UnionImpl.java | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 96b25576b..f03c5078f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ *.ipr *.iws +# VSCode project files +**/.vscode/ + # Additional tools .clover/ diff --git a/pom.xml b/pom.xml index 949152680..abcbd0c54 100644 --- a/pom.xml +++ b/pom.xml @@ -75,7 +75,7 @@ under the License. - 2.2.0 + 4.0.0 7.10.0 diff --git a/src/main/java/org/apache/datasketches/theta/UnionImpl.java b/src/main/java/org/apache/datasketches/theta/UnionImpl.java index 3ea5ca557..7288365b9 100644 --- a/src/main/java/org/apache/datasketches/theta/UnionImpl.java +++ b/src/main/java/org/apache/datasketches/theta/UnionImpl.java @@ -321,7 +321,7 @@ public void union(final Sketch sketchIn) { if (sketchIn.isOrdered() && (sketchIn instanceof CompactSketch)) { //Use early stop //Ordered, thus compact if (sketchIn.hasMemory()) { - final Memory skMem = ((CompactSketch) sketchIn).getMemory(); + final Memory skMem = sketchIn.getMemory(); final int preambleLongs = skMem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F; for (int i = 0; i < curCountIn; i++ ) { final int offsetBytes = preambleLongs + i << 3; From 4a061fe70a9d94aa296dc8c6b8e9b36f25bef466 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Sun, 17 Nov 2024 12:49:44 -0800 Subject: [PATCH 24/88] update GHA, pom --- ...anual-os-matrix.yml => auto-os-matrix.yml} | 22 ++++++++++++------- .github/workflows/codeql-analysis.yml | 8 +++---- .github/workflows/javadoc.yml | 22 ++++++++++++++----- .github/workflows/manual-coverage.yml | 12 +++++----- pom.xml | 2 +- 5 files changed, 42 insertions(+), 24 deletions(-) rename .github/workflows/{manual-os-matrix.yml => auto-os-matrix.yml} (85%) diff --git a/.github/workflows/manual-os-matrix.yml b/.github/workflows/auto-os-matrix.yml similarity index 85% rename from .github/workflows/manual-os-matrix.yml rename to .github/workflows/auto-os-matrix.yml index 2d5537841..a8afc3b6b 100644 --- a/.github/workflows/manual-os-matrix.yml +++ b/.github/workflows/auto-os-matrix.yml @@ -1,19 +1,23 @@ -name: DataSketches-Java Manual OS Matrix Test & Install +name: DataSketches-Java Auto OS Matrix Test & Install on: + pull_request: + push: + branches: [ master ] workflow_dispatch: env: - MAVEN_OPTS: -Xmx4g -Xms1g + MAVEN_OPTS: -Xmx1g -Xms1g jobs: build: name: Build, Test, Install - runs-on: ${{matrix.os}} + strategy: fail-fast: false + matrix: - jdk: [ 8, 11 ] + jdk: [ 17 ] os: [ windows-latest, ubuntu-latest, macos-latest ] include: - os: windows-latest @@ -26,31 +30,33 @@ jobs: skip_javadoc: -Dmaven.javadoc.skip=true skip_gpg: -Dgpg.skip=true + runs-on: ${{matrix.os}} + env: JDK_VERSION: ${{ matrix.jdk }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: build-${{ runner.os }}-maven- - name: Install Matrix JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.jdk }} distribution: 'temurin' java-package: jdk architecture: x64 # Architecture options: x86, x64, armv7, aarch64, ppc64le -# setup-java@v3 has a "with cache" option +# setup-java@v4 has a "with cache" option - name: Echo Java Version run: > diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7cb4c8976..51726d6d7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -28,11 +28,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -46,7 +46,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@v4 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -59,6 +59,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/javadoc.yml b/.github/workflows/javadoc.yml index 6033d3273..d1c554291 100644 --- a/.github/workflows/javadoc.yml +++ b/.github/workflows/javadoc.yml @@ -2,22 +2,34 @@ name: JavaDoc on: push: - branches: - - master + branches: [7.0.X, master] workflow_dispatch: jobs: javadoc: runs-on: ubuntu-latest + steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - name: Setup Java + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Echo Java Version + run: java -version + - name: Generate JavaDoc run: mvn javadoc:javadoc + - name: Deploy JavaDoc - uses: JamesIves/github-pages-deploy-action@5dc1d5a192aeb5ab5b7d5a77b7d36aea4a7f5c92 + uses: JamesIves/github-pages-deploy-action@4.6.8 + #uses: JamesIves/github-pages-deploy-action@5dc1d5a192aeb5ab5b7d5a77b7d36aea4a7f5c92 with: token: ${{ secrets.GITHUB_TOKEN }} - folder: target/site/apidocs + folder: target/reports/apidocs target-folder: docs/${{ github.ref_name }} branch: gh-pages diff --git a/.github/workflows/manual-coverage.yml b/.github/workflows/manual-coverage.yml index e7b6ebc81..79b6006b3 100644 --- a/.github/workflows/manual-coverage.yml +++ b/.github/workflows/manual-coverage.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [ 8 ] + jdk: [ 17 ] os: [ ubuntu-latest ] include: # - os: windows-latest @@ -31,33 +31,33 @@ jobs: steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: build-${{ runner.os }}-maven- - name: Install Matrix JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.jdk }} distribution: 'temurin' java-package: jdk architecture: x64 # Architecture options: x86, x64, armv7, aarch64, ppc64le -# setup-java@v3 has a "with cache" option +# setup-java@v4 has a "with cache" option - name: Echo Java Version run: > java -version - name: Test, Package, Verify, Coverage Report - if: ${{ matrix.jdk == 8 && success() }} + if: ${{ matrix.jdk == 17 && success() }} run: mvn verify coveralls:report -B -DrepoToken=${{secrets.coveralls_token}} diff --git a/pom.xml b/pom.xml index d6635e6e4..c843c9835 100644 --- a/pom.xml +++ b/pom.xml @@ -83,7 +83,7 @@ under the License. - 4.0.0 + 4.1.0 7.10.2 From d1853a9774e3a2dcb32c088fda5acf655a96a92a Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 18 Nov 2024 14:40:37 -0800 Subject: [PATCH 25/88] Upload fixes as a result of CodeQL run. --- .github/workflows/codeql-analysis.yml | 12 +-- .github/workflows/javadoc.yml | 4 +- .../filters/bloomfilter/package-info.java | 4 +- .../apache/datasketches/hll/package-info.java | 2 +- .../quantiles/CompactDoublesSketch.java | 5 - .../datasketches/quantiles/DoublesSketch.java | 2 +- .../quantiles/UpdateDoublesSketch.java | 7 ++ .../quantilescommon/QuantilesAPI.java | 11 +++ .../datasketches/req/BaseReqSketch.java | 14 ++- .../thetacommon/SetOperationCornerCases.java | 93 +++++++++++++++++++ .../tuple/SerializerDeserializer.java | 13 ++- .../QuantilesSketchCrossLanguageTest.java | 2 +- 12 files changed, 149 insertions(+), 20 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 51726d6d7..376d69404 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,13 +1,13 @@ name: "CodeQL" on: - push: - branches: [ 'master' ] - pull_request: +# push: +# branches: [ 'master' ] +# pull_request: # The branches below must be a subset of the branches above - branches: [ 'master' ] - schedule: - - cron: '10 17 * * 4' +# branches: [ 'master' ] +# schedule: +# - cron: '10 17 * * 4' workflow_dispatch: jobs: diff --git a/.github/workflows/javadoc.yml b/.github/workflows/javadoc.yml index d1c554291..5487b9e7b 100644 --- a/.github/workflows/javadoc.yml +++ b/.github/workflows/javadoc.yml @@ -26,8 +26,8 @@ jobs: run: mvn javadoc:javadoc - name: Deploy JavaDoc - uses: JamesIves/github-pages-deploy-action@4.6.8 - #uses: JamesIves/github-pages-deploy-action@5dc1d5a192aeb5ab5b7d5a77b7d36aea4a7f5c92 + uses: JamesIves/github-pages-deploy-action@v4.6.8 + #uses: JamesIves/github-pages-deploy-action@881db5376404c5c8d621010bcbec0310b58d5e29 #alternate with: token: ${{ secrets.GITHUB_TOKEN }} folder: target/reports/apidocs diff --git a/src/main/java/org/apache/datasketches/filters/bloomfilter/package-info.java b/src/main/java/org/apache/datasketches/filters/bloomfilter/package-info.java index 8a752e168..5cc6f28fe 100644 --- a/src/main/java/org/apache/datasketches/filters/bloomfilter/package-info.java +++ b/src/main/java/org/apache/datasketches/filters/bloomfilter/package-info.java @@ -16,5 +16,7 @@ * specific language governing permissions and limitations * under the License. */ - +/** + * BloomFilter package + */ package org.apache.datasketches.filters.bloomfilter; diff --git a/src/main/java/org/apache/datasketches/hll/package-info.java b/src/main/java/org/apache/datasketches/hll/package-info.java index 9ffafa7bd..625cb011c 100644 --- a/src/main/java/org/apache/datasketches/hll/package-info.java +++ b/src/main/java/org/apache/datasketches/hll/package-info.java @@ -18,7 +18,7 @@ */ /** - *

The DataSketches™ HLL sketch family package

+ *

The DataSketches™ HLL sketch family package

* {@link org.apache.datasketches.hll.HllSketch HllSketch} and {@link org.apache.datasketches.hll.Union Union} * are the public facing classes of this high performance implementation of Phillipe Flajolet's * HyperLogLog algorithm[1] but with significantly improved error behavior and important features that can be diff --git a/src/main/java/org/apache/datasketches/quantiles/CompactDoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/CompactDoublesSketch.java index f6df9e87e..18e05d315 100644 --- a/src/main/java/org/apache/datasketches/quantiles/CompactDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/CompactDoublesSketch.java @@ -20,7 +20,6 @@ package org.apache.datasketches.quantiles; import org.apache.datasketches.common.SketchesStateException; -import org.apache.datasketches.memory.Memory; /** * Compact sketches are inherently read only. @@ -31,10 +30,6 @@ public abstract class CompactDoublesSketch extends DoublesSketch { super(k); } - public static CompactDoublesSketch heapify(final Memory srcMem) { - return HeapCompactDoublesSketch.heapifyInstance(srcMem); - } - @Override boolean isCompact() { return true; diff --git a/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java index 03f84116a..49e7c5fbb 100644 --- a/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java @@ -140,7 +140,7 @@ public static final DoublesSketchBuilder builder() { */ public static DoublesSketch heapify(final Memory srcMem) { if (checkIsCompactMemory(srcMem)) { - return CompactDoublesSketch.heapify(srcMem); + return HeapCompactDoublesSketch.heapifyInstance(srcMem); } return UpdateDoublesSketch.heapify(srcMem); } diff --git a/src/main/java/org/apache/datasketches/quantiles/UpdateDoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/UpdateDoublesSketch.java index 155001c87..f56c3b352 100644 --- a/src/main/java/org/apache/datasketches/quantiles/UpdateDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/UpdateDoublesSketch.java @@ -49,6 +49,13 @@ public static UpdateDoublesSketch wrap(final WritableMemory srcMem) { @Override public abstract void update(double item); + /** + * Factory heapify takes a compact sketch image in Memory and instantiates an on-heap sketch. + * The resulting sketch will not retain any link to the source Memory. + * @param srcMem a compact Memory image of a sketch serialized by this sketch. + * See Memory + * @return a heap-based sketch based on the given Memory. + */ public static UpdateDoublesSketch heapify(final Memory srcMem) { return HeapUpdateDoublesSketch.heapifyInstance(srcMem); } diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java index 9943fefee..3dc0651a7 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java @@ -205,11 +205,22 @@ @SuppressWarnings("javadoc") public interface QuantilesAPI { + /** The sketch must not be empty for this operation. */ static String EMPTY_MSG = "The sketch must not be empty for this operation. "; + + /** Unsupported operation for this Sketch Type. */ static String UNSUPPORTED_MSG = "Unsupported operation for this Sketch Type. "; + + /** Sketch does not have just one item. */ static String NOT_SINGLE_ITEM_MSG = "Sketch does not have just one item. "; + + /** MemoryRequestServer must not be null. */ static String MEM_REQ_SVR_NULL_MSG = "MemoryRequestServer must not be null. "; + + /** Target sketch is Read Only, cannot write. */ static String TGT_IS_READ_ONLY_MSG = "Target sketch is Read Only, cannot write. "; + + /** A sketch cannot merge with itself. */ static String SELF_MERGE_MSG = "A sketch cannot merge with itself. "; /** diff --git a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java index 2460308fc..05d4d3ce2 100644 --- a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java +++ b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java @@ -89,11 +89,23 @@ public static double getRSE(final int k, final double rank, final boolean hra, f @Override public abstract float getQuantileLowerBound(double rank); + /** + * Gets an approximate lower bound of the quantile associated with the given rank. + * @param rank the given normalized rank, a number between 0 and 1.0. + * @param numStdDev the number of standard deviations. Must be 1, 2, or 3. + * @return an approximate lower bound quantile, if it exists. + */ public abstract float getQuantileLowerBound(double rank, int numStdDev); @Override public abstract float getQuantileUpperBound(double rank); + /** + * Gets an approximate upper bound of the quantile associated with the given rank. + * @param rank the given normalized rank, a number between 0 and 1.0. + * @param numStdDev the number of standard deviations. Must be 1, 2, or 3. + * @return an approximate upper bound quantile, if it exists. + */ public abstract float getQuantileUpperBound(double rank, int numStdDev); @Override @@ -101,7 +113,7 @@ public static double getRSE(final int k, final double rank, final boolean hra, f /** * Gets an approximate lower bound rank of the given normalized rank. - * @param rank the given rank, a number between 0 and 1.0. + * @param rank the given normalized rank, a number between 0 and 1.0. * @param numStdDev the number of standard deviations. Must be 1, 2, or 3. * @return an approximate lower bound rank. */ diff --git a/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java b/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java index 20dd6ee7d..72a98565d 100644 --- a/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java +++ b/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java @@ -46,10 +46,18 @@ private IntersectAction(final String actionId, final String actionDescription) { this.actionDescription = actionDescription; } + /** + * Gets the Action ID + * @return the actionId + */ public String getActionId() { return actionId; } + /** + * Gets the Action Description + * @return the actionDescription + */ public String getActionDescription() { return actionDescription; } @@ -72,24 +80,42 @@ private AnotbAction(final String actionId, final String actionDescription) { this.actionDescription = actionDescription; } + /** + * Gets the Action ID + * @return the actionId + */ public String getActionId() { return actionId; } + /** + * Gets the action description + * @return the action description + */ public String getActionDescription() { return actionDescription; } } + /** List of union actions */ public enum UnionAction { + /** Sketch A Exactly */ SKETCH_A("A", "Sketch A Exactly"), + /** Trim Sketch A by MinTheta */ TRIM_A("TA", "Trim Sketch A by MinTheta"), + /** Sketch B Exactly */ SKETCH_B("B", "Sketch B Exactly"), + /** Trim Sketch B by MinTheta */ TRIM_B("TB", "Trim Sketch B by MinTheta"), + /** Degenerate{MinTheta, 0, F} */ DEGEN_MIN_0_F("D", "Degenerate{MinTheta, 0, F}"), + /** Degenerate{ThetaA, 0, F} */ DEGEN_THA_0_F("DA", "Degenerate{ThetaA, 0, F}"), + /** Degenerate{ThetaB, 0, F} */ DEGEN_THB_0_F("DB", "Degenerate{ThetaB, 0, F}"), + /** Empty{1.0, 0, T} */ EMPTY_1_0_T("E", "Empty{1.0, 0, T}"), + /** Full Union */ FULL_UNION("N", "Full Union"); private String actionId; @@ -100,49 +126,74 @@ private UnionAction(final String actionId, final String actionDescription) { this.actionDescription = actionDescription; } + /** + * Gets the action ID + * @return the actionId + */ public String getActionId() { return actionId; } + /** + * Gets the action description + * @return the actionDescription + */ public String getActionDescription() { return actionDescription; } } + /** List of corner cases */ public enum CornerCase { + /** Empty Empty */ Empty_Empty(055, "A{ 1.0, 0, T} ; B{ 1.0, 0, T}", IntersectAction.EMPTY_1_0_T, AnotbAction.EMPTY_1_0_T, UnionAction.EMPTY_1_0_T), + /** Empty Exact */ Empty_Exact(056, "A{ 1.0, 0, T} ; B{ 1.0,>0, F}", IntersectAction.EMPTY_1_0_T, AnotbAction.EMPTY_1_0_T, UnionAction.SKETCH_B), + /** Empty Estimation */ Empty_Estimation(052, "A{ 1.0, 0, T} ; B{<1.0,>0, F", IntersectAction.EMPTY_1_0_T, AnotbAction.EMPTY_1_0_T, UnionAction.SKETCH_B), + /** Empty Degen */ Empty_Degen(050, "A{ 1.0, 0, T} ; B{<1.0, 0, F}", IntersectAction.EMPTY_1_0_T, AnotbAction.EMPTY_1_0_T, UnionAction.DEGEN_THB_0_F), + /** Exact Empty */ Exact_Empty(065, "A{ 1.0,>0, F} ; B{ 1.0, 0, T}", IntersectAction.EMPTY_1_0_T, AnotbAction.SKETCH_A, UnionAction.SKETCH_A), + /** Exact Exact */ Exact_Exact(066, "A{ 1.0,>0, F} ; B{ 1.0,>0, F}", IntersectAction.FULL_INTERSECT, AnotbAction.FULL_ANOTB, UnionAction.FULL_UNION), + /** Exact Estimation */ Exact_Estimation(062, "A{ 1.0,>0, F} ; B{<1.0,>0, F}", IntersectAction.FULL_INTERSECT, AnotbAction.FULL_ANOTB, UnionAction.FULL_UNION), + /** Exact Degen */ Exact_Degen(060, "A{ 1.0,>0, F} ; B{<1.0, 0, F}", IntersectAction.DEGEN_MIN_0_F, AnotbAction.TRIM_A, UnionAction.TRIM_A), + /** Estimation_Empty */ Estimation_Empty(025, "A{<1.0,>0, F} ; B{ 1.0, 0, T}", IntersectAction.EMPTY_1_0_T, AnotbAction.SKETCH_A, UnionAction.SKETCH_A), + /** Estimation_Exact */ Estimation_Exact(026, "A{<1.0,>0, F} ; B{ 1.0,>0, F}", IntersectAction.FULL_INTERSECT, AnotbAction.FULL_ANOTB, UnionAction.FULL_UNION), + /** Estimation_Estimation */ Estimation_Estimation(022, "A{<1.0,>0, F} ; B{<1.0,>0, F}", IntersectAction.FULL_INTERSECT, AnotbAction.FULL_ANOTB, UnionAction.FULL_UNION), + /** Estimation_Degen */ Estimation_Degen(020, "A{<1.0,>0, F} ; B{<1.0, 0, F}", IntersectAction.DEGEN_MIN_0_F, AnotbAction.TRIM_A, UnionAction.TRIM_A), + /** Degen_Empty */ Degen_Empty(005, "A{<1.0, 0, F} ; B{ 1.0, 0, T}", IntersectAction.EMPTY_1_0_T, AnotbAction.DEGEN_THA_0_F, UnionAction.DEGEN_THA_0_F), + /** Degen_Exact */ Degen_Exact(006, "A{<1.0, 0, F} ; B{ 1.0,>0, F}", IntersectAction.DEGEN_MIN_0_F, AnotbAction.DEGEN_THA_0_F, UnionAction.TRIM_B), + /** Degen_Estimation */ Degen_Estimation(002, "A{<1.0, 0, F} ; B{<1.0,>0, F}", IntersectAction.DEGEN_MIN_0_F, AnotbAction.DEGEN_MIN_0_F, UnionAction.TRIM_B), + /** Degen_Degen */ Degen_Degen(000, "A{<1.0, 0, F} ; B{<1.0, 0, F}", IntersectAction.DEGEN_MIN_0_F, AnotbAction.DEGEN_MIN_0_F, UnionAction.DEGEN_MIN_0_F); @@ -168,27 +219,52 @@ private CornerCase(final int caseId, final String caseDescription, this.unionAction = unionAction; } + /** + * Gets the case ID + * @return the caseId + */ public int getId() { return caseId; } + /** + * Gets the case description + * @return the caseDescription + */ public String getCaseDescription() { return caseDescription; } + /** + * Gets the intersect action + * @return the intersectAction + */ public IntersectAction getIntersectAction() { return intersectAction; } + /** + * Gets the AnotB action + * @return the anotbAction + */ public AnotbAction getAnotbAction() { return anotbAction; } + /** + * Gets the union action + * @return the unionAction + */ public UnionAction getUnionAction() { return unionAction; } //See checkById test in /tuple/MiscTest. + /** + * Converts caseId to CornerCaseId + * @param id the case ID + * @return the Corner Case ID + */ public static CornerCase caseIdToCornerCase(final int id) { final CornerCase cc = caseIdToCornerCaseMap.get(id); if (cc == null) { @@ -198,12 +274,29 @@ public static CornerCase caseIdToCornerCase(final int id) { } } //end of enum CornerCase + /** + * Creates the CornerCase ID + * @param thetaLongA the theta of A as a long + * @param countA the count of A + * @param emptyA true if A is empty + * @param thetaLongB the theta of B as a long + * @param countB the count of B + * @param emptyB true if B is empty + * @return the Corner Case ID + */ public static int createCornerCaseId( final long thetaLongA, final int countA, final boolean emptyA, final long thetaLongB, final int countB, final boolean emptyB) { return (sketchStateId(emptyA, countA, thetaLongA) << 3) | sketchStateId(emptyB, countB, thetaLongB); } + /** + * Returns the sketch state ID + * @param isEmpty true if empty + * @param numRetained the number of items retained + * @param thetaLong the value of theta as a long + * @return the sketch state ID + */ public static int sketchStateId(final boolean isEmpty, final int numRetained, final long thetaLong) { // assume thetaLong = MAX if empty return (((thetaLong == MAX) || isEmpty) ? 4 : 0) | ((numRetained > 0) ? 2 : 0) | (isEmpty ? 1 : 0); diff --git a/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java b/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java index 44d1d9cc0..9b0ca33cb 100644 --- a/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java +++ b/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java @@ -32,8 +32,17 @@ public final class SerializerDeserializer { * Defines the sketch classes that this SerializerDeserializer can handle. */ @SuppressWarnings("javadoc") - public static enum SketchType { QuickSelectSketch, CompactSketch, ArrayOfDoublesQuickSelectSketch, - ArrayOfDoublesCompactSketch, ArrayOfDoublesUnion } + public static enum SketchType { + /** QuickSelectSketch */ + QuickSelectSketch, + /** CompactSketch */ + CompactSketch, + /** ArrayOfDoublesQuickSelectSketch */ + ArrayOfDoublesQuickSelectSketch, + /** ArrayOfDoublesCompactSketch */ + ArrayOfDoublesCompactSketch, + /** ArrayOfDoublesUnion */ + ArrayOfDoublesUnion } static final int TYPE_BYTE_OFFSET = 3; diff --git a/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java index 8c80f4399..68347ffb8 100644 --- a/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java @@ -249,7 +249,7 @@ private static void getAndCheck(String ver, int n, double quantile) { Assert.assertEquals(q2, quantile, 0.0); // same thing with compact sketch - qs2 = CompactDoublesSketch.heapify(srcMem); + qs2 = HeapCompactDoublesSketch.heapifyInstance(srcMem); //Test the quantile q2 = qs2.getQuantile(nf, EXCLUSIVE); println("New Median: " + q2); From f42af451b38c096d0d4af5f45814c86ef252e78e Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 18 Nov 2024 15:36:57 -0800 Subject: [PATCH 26/88] Update again for coverage reported errors. --- .../java/org/apache/datasketches/hll/package-info.java | 2 +- .../thetacommon/SetOperationCornerCases.java | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/hll/package-info.java b/src/main/java/org/apache/datasketches/hll/package-info.java index 625cb011c..a17a9f646 100644 --- a/src/main/java/org/apache/datasketches/hll/package-info.java +++ b/src/main/java/org/apache/datasketches/hll/package-info.java @@ -18,7 +18,7 @@ */ /** - *

The DataSketches™ HLL sketch family package

+ *

The DataSketches™ HLL sketch family package

* {@link org.apache.datasketches.hll.HllSketch HllSketch} and {@link org.apache.datasketches.hll.Union Union} * are the public facing classes of this high performance implementation of Phillipe Flajolet's * HyperLogLog algorithm[1] but with significantly improved error behavior and important features that can be diff --git a/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java b/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java index 72a98565d..d9fda48bb 100644 --- a/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java +++ b/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java @@ -34,8 +34,11 @@ public class SetOperationCornerCases { /** Intersection actions */ public enum IntersectAction { + /** Degenerate{MinTheta, 0, F} */ DEGEN_MIN_0_F("D", "Degenerate{MinTheta, 0, F}"), + /** Empty{1.0, 0, T */ EMPTY_1_0_T("E", "Empty{1.0, 0, T}"), + /** Full Intersect */ FULL_INTERSECT("I", "Full Intersect"); private String actionId; @@ -65,11 +68,17 @@ public String getActionDescription() { /** A not B actions */ public enum AnotbAction { + /** Sketch A Exact */ SKETCH_A("A", "Sketch A Exactly"), + /** Trim Sketch A by MinTheta */ TRIM_A("TA", "Trim Sketch A by MinTheta"), + /** Degenerate{MinTheta, 0, F} */ DEGEN_MIN_0_F("D", "Degenerate{MinTheta, 0, F}"), + /** Degenerate{ThetaA, 0, F} */ DEGEN_THA_0_F("DA", "Degenerate{ThetaA, 0, F}"), + /** Empty{1.0, 0, T} */ EMPTY_1_0_T("E", "Empty{1.0, 0, T}"), + /** Full AnotB */ FULL_ANOTB("N", "Full AnotB"); private String actionId; From 34aaa8baf679cccdac2ea6bdefef46870e5bc004 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Tue, 10 Dec 2024 15:35:28 -0800 Subject: [PATCH 27/88] This huge update does not impact any code. It is mostly Javadoc updates. There are also a few pom updates and some updates to the GH Action workflows. --- .github/workflows/auto-jdk-matrix.yml | 2 +- .github/workflows/auto-os-matrix.yml | 6 +- .github/workflows/javadoc.yml | 7 +- .github/workflows/manual-coverage.yml | 71 -------------- pom.xml | 93 +++++++++++-------- .../filters/bloomfilter/BloomFilter.java | 4 +- .../bloomfilter/BloomFilterBuilder.java | 4 +- .../datasketches/frequencies/ItemsSketch.java | 4 +- .../datasketches/frequencies/LongsSketch.java | 4 +- .../frequencies/PreambleUtil.java | 1 + .../apache/datasketches/hash/MurmurHash3.java | 2 - .../datasketches/hash/package-info.java | 3 +- .../datasketches/kll/KllDoublesHelper.java | 1 + .../datasketches/kll/KllDoublesSketch.java | 1 + .../datasketches/kll/KllFloatsHelper.java | 1 + .../datasketches/kll/KllFloatsSketch.java | 1 + .../datasketches/kll/KllItemsHelper.java | 1 + .../datasketches/kll/KllLongsHelper.java | 1 + .../datasketches/kll/KllLongsSketch.java | 1 + .../datasketches/quantiles/DoublesSketch.java | 1 + .../datasketches/quantiles/package-info.java | 3 +- .../quantilescommon/DoublesSortedView.java | 8 +- .../quantilescommon/FloatsSortedView.java | 8 +- .../quantilescommon/GenericSortedView.java | 8 +- .../quantilescommon/LongsSortedView.java | 8 +- .../quantilescommon/QuantilesAPI.java | 4 +- .../quantilescommon/QuantilesDoublesAPI.java | 8 +- .../quantilescommon/QuantilesFloatsAPI.java | 8 +- .../quantilescommon/QuantilesGenericAPI.java | 8 +- .../quantilescommon/QuantilesLongsAPI.java | 8 +- .../datasketches/req/BaseReqSketch.java | 1 + .../org/apache/datasketches/req/ReqSerDe.java | 3 +- .../sampling/ReservoirLongsUnion.java | 7 +- .../datasketches/sampling/ReservoirSize.java | 1 + .../datasketches/sampling/package-info.java | 4 +- .../datasketches/theta/PreambleUtil.java | 3 +- .../datasketches/theta/UpdateSketch.java | 1 + .../org/apache/datasketches/tuple/Union.java | 3 +- .../tuple/adouble/DoubleSummary.java | 4 + .../tuple/aninteger/IntegerSummary.java | 4 + .../DirectArrayOfDoublesCompactSketch.java | 1 + .../DirectArrayOfDoublesIntersection.java | 3 +- ...DirectArrayOfDoublesQuickSelectSketch.java | 1 + .../DirectArrayOfDoublesSketchIterator.java | 1 + .../DirectArrayOfDoublesUnion.java | 1 + tools/SketchesCheckstyle.xml | 2 +- 46 files changed, 146 insertions(+), 174 deletions(-) delete mode 100644 .github/workflows/manual-coverage.yml diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml index ce15bb57e..6e280f123 100644 --- a/.github/workflows/auto-jdk-matrix.yml +++ b/.github/workflows/auto-jdk-matrix.yml @@ -3,7 +3,7 @@ name: DataSketches-Java Auto JDK Matrix Test & Install on: pull_request: push: - branches: [ master ] + branches: [ master, main ] workflow_dispatch: env: diff --git a/.github/workflows/auto-os-matrix.yml b/.github/workflows/auto-os-matrix.yml index a8afc3b6b..d9f5bc4c5 100644 --- a/.github/workflows/auto-os-matrix.yml +++ b/.github/workflows/auto-os-matrix.yml @@ -3,7 +3,7 @@ name: DataSketches-Java Auto OS Matrix Test & Install on: pull_request: push: - branches: [ master ] + branches: [ master, main ] workflow_dispatch: env: @@ -55,8 +55,6 @@ jobs: distribution: 'temurin' java-package: jdk architecture: x64 -# Architecture options: x86, x64, armv7, aarch64, ppc64le -# setup-java@v4 has a "with cache" option - name: Echo Java Version run: > @@ -75,6 +73,8 @@ jobs: -D skipTests=true ${{matrix.os.skip_gpg}} +# Architecture options: x86, x64, armv7, aarch64, ppc64le +# setup-java@v4 has a "with cache" option # Lifecycle: validate, compile, test, package, verify, install, deploy # -B batch mode # -V show Version without stopping diff --git a/.github/workflows/javadoc.yml b/.github/workflows/javadoc.yml index 5487b9e7b..04d70e25a 100644 --- a/.github/workflows/javadoc.yml +++ b/.github/workflows/javadoc.yml @@ -2,7 +2,7 @@ name: JavaDoc on: push: - branches: [7.0.X, master] + branches: [ master, main ] workflow_dispatch: jobs: @@ -22,12 +22,15 @@ jobs: - name: Echo Java Version run: java -version + - name: Print Current workflow + run: > + cat .github/workflows/javadoc.yml + - name: Generate JavaDoc run: mvn javadoc:javadoc - name: Deploy JavaDoc uses: JamesIves/github-pages-deploy-action@v4.6.8 - #uses: JamesIves/github-pages-deploy-action@881db5376404c5c8d621010bcbec0310b58d5e29 #alternate with: token: ${{ secrets.GITHUB_TOKEN }} folder: target/reports/apidocs diff --git a/.github/workflows/manual-coverage.yml b/.github/workflows/manual-coverage.yml deleted file mode 100644 index 79b6006b3..000000000 --- a/.github/workflows/manual-coverage.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Datasketches-Java Manual Coverage Report - -on: - workflow_dispatch: - -env: - MAVEN_OPTS: -Xmx4g -Xms1g - -jobs: - build: - name: Build, Test, Coverage - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - jdk: [ 17 ] - os: [ ubuntu-latest ] - include: -# - os: windows-latest -# skip_javadoc: "`-Dmaven`.javadoc`.skip=true" -# skip_gpg: "`-Dgpg`.skip=true" - - os: ubuntu-latest - skip_javadoc: -Dmaven.javadoc.skip=true - skip_gpg: -Dgpg.skip=true -# - os: macos-latest -# skip_javadoc: -Dmaven.javadoc.skip=true -# skip_gpg: -Dgpg.skip=true - - env: - JDK_VERSION: ${{ matrix.jdk }} - - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Cache local Maven repository - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: build-${{ runner.os }}-maven- - - - name: Install Matrix JDK - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.jdk }} - distribution: 'temurin' - java-package: jdk - architecture: x64 -# Architecture options: x86, x64, armv7, aarch64, ppc64le -# setup-java@v4 has a "with cache" option - - - name: Echo Java Version - run: > - java -version - - - name: Test, Package, Verify, Coverage Report - if: ${{ matrix.jdk == 17 && success() }} - run: - mvn verify coveralls:report -B - -DrepoToken=${{secrets.coveralls_token}} - ${{matrix.os.skip_javadoc}} - ${{matrix.os.skip_gpg}} - -# Lifecycle: validate, compile, test, package, verify, install, deploy -# Coverage reports are available after the verify phase -# -B batch mode -# -V show Version without stopping -# -q quiet, only show errors diff --git a/pom.xml b/pom.xml index c843c9835..ccff0122d 100644 --- a/pom.xml +++ b/pom.xml @@ -95,9 +95,10 @@ under the License. 3.6.3 17 + --add-modules=jdk.incubator.foreign ${java.version} ${java.version} - -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 --add-modules=jdk.incubator.foreign + -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 ${add-modules} UTF-8 ${charset.encoding} ${charset.encoding} @@ -105,30 +106,31 @@ under the License. yyyy-MM-dd'T'HH-mm-ss'Z' - 3.7.1 - 3.13.0 - 3.1.3 - 3.5.0 - 3.2.5 - 3.4.2 - 3.8.0 - 3.1.1 - 3.2.0 - 3.3.1 - 3.4.0 - - 4.9.10 - - 0.16.1 - - - 4.3.0 - - 0.8.12 - - 2.17.1 - - 1.0.0 + 3.7.1 + 3.13.0 + 3.1.3 + 3.5.0 + 3.2.7 + 3.4.2 + 3.11.1 + 3.1.1 + 3.2.0 + 3.3.1 + 3.5.2 + 3.2.0 + + 4.9.10 + + 0.16.1 + + + 4.3.0 + + 0.8.12 + + 2.17.1 + + 1.0.0 @@ -145,13 +147,6 @@ under the License. ${testng.version} test - @@ -169,7 +164,7 @@ under the License. ${maven-compiler-plugin.version} - --add-modules=jdk.incubator.foreign + ${add-modules} @@ -195,10 +190,10 @@ under the License. - [17,18) + ${java.version} - [${maven.version},) + [${maven.version},4.0.0) @@ -243,7 +238,7 @@ under the License. true public - --add-modules=jdk.incubator.foreign + ${add-modules} @@ -289,7 +284,7 @@ under the License. maven-surefire-plugin ${maven-surefire-failsafe-plugins.version} - --add-modules=jdk.incubator.foreign + ${add-modules} false false true @@ -298,6 +293,26 @@ under the License. + + org.apache.maven.plugins + maven-toolchains-plugin + ${maven-toolchains-plugin.version} + + + + toolchain + + + + + + + ${java.version} + + + + + org.apache.rat apache-rat-plugin @@ -402,6 +417,10 @@ under the License. org.apache.maven.plugins maven-surefire-plugin + + org.apache.maven.plugins + maven-toolchains-plugin + org.apache.rat apache-rat-plugin diff --git a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java index eafe88340..171fc2cfb 100644 --- a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java +++ b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java @@ -33,8 +33,8 @@ import org.apache.datasketches.memory.XxHash; /** - *

A Bloom filter is a data structure that can be used for probabilistic - * set membership.

+ * A Bloom filter is a data structure that can be used for probabilistic + * set membership. * *

When querying a Bloom filter, there are no false positives. Specifically: * When querying an item that has already been inserted to the filter, the filter will diff --git a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilterBuilder.java b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilterBuilder.java index f865a3350..ee17a9918 100644 --- a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilterBuilder.java +++ b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilterBuilder.java @@ -25,8 +25,8 @@ import org.apache.datasketches.memory.WritableMemory; /** - *

This class provides methods to help estimate the correct parameters when - * creating a Bloom filter, and methods to create the filter using those values.

+ * This class provides methods to help estimate the correct parameters when + * creating a Bloom filter, and methods to create the filter using those values. * *

The underlying math is described in the * diff --git a/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java b/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java index 7c5bc770d..b2648b943 100644 --- a/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java +++ b/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java @@ -55,10 +55,10 @@ import org.apache.datasketches.memory.WritableMemory; /** - *

This sketch is useful for tracking approximate frequencies of items of type <T> + * This sketch is useful for tracking approximate frequencies of items of type <T> * with optional associated counts (<T> item, long count) that are members of a * multiset of such items. The true frequency of an item is defined to be the sum of associated - * counts.

+ * counts. * *

This implementation provides the following capabilities:

*
    diff --git a/src/main/java/org/apache/datasketches/frequencies/LongsSketch.java b/src/main/java/org/apache/datasketches/frequencies/LongsSketch.java index 30452a1b5..7d1990b62 100644 --- a/src/main/java/org/apache/datasketches/frequencies/LongsSketch.java +++ b/src/main/java/org/apache/datasketches/frequencies/LongsSketch.java @@ -54,9 +54,9 @@ import org.apache.datasketches.memory.WritableMemory; /** - *

    This sketch is useful for tracking approximate frequencies of long items with optional + * This sketch is useful for tracking approximate frequencies of long items with optional * associated counts (long item, long count) that are members of a multiset of - * such items. The true frequency of an item is defined to be the sum of associated counts.

    + * such items. The true frequency of an item is defined to be the sum of associated counts. * *

    This implementation provides the following capabilities:

    *
      diff --git a/src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java b/src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java index 785c211f2..b5db6025c 100644 --- a/src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java @@ -31,6 +31,7 @@ /** * This class defines the preamble data structure and provides basic utilities for some of the key * fields. + * *

      * The intent of the design of this class was to isolate the detailed knowledge of the bit and byte * layout of the serialized form of the sketches derived from the Sketch class into one place. This diff --git a/src/main/java/org/apache/datasketches/hash/MurmurHash3.java b/src/main/java/org/apache/datasketches/hash/MurmurHash3.java index 9fb2ab948..a708e0077 100644 --- a/src/main/java/org/apache/datasketches/hash/MurmurHash3.java +++ b/src/main/java/org/apache/datasketches/hash/MurmurHash3.java @@ -29,10 +29,8 @@ import org.apache.datasketches.memory.Memory; /** - *

      * The MurmurHash3 is a fast, non-cryptographic, 128-bit hash function that has * excellent avalanche and 2-way bit independence properties. - *

      * *

      * Austin Appleby's C++ diff --git a/src/main/java/org/apache/datasketches/hash/package-info.java b/src/main/java/org/apache/datasketches/hash/package-info.java index 2d97afeb6..5744b2776 100644 --- a/src/main/java/org/apache/datasketches/hash/package-info.java +++ b/src/main/java/org/apache/datasketches/hash/package-info.java @@ -18,12 +18,11 @@ */ /** - *

      The hash package contains a high-performing and extended Java implementations + * The hash package contains a high-performing and extended Java implementations * of Austin Appleby's 128-bit MurmurHash3 hash function originally coded in C. * This core MurmurHash3.java class is used throughout many of the sketch classes for consistency * and as long as the user specifies the same seed will result in coordinated hash operations. * This package also contains an adaptor class that extends the basic class with more functions * commonly associated with hashing. - *

      */ package org.apache.datasketches.hash; diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java index 67035b45f..acbecdf07 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java @@ -312,6 +312,7 @@ private static void randomlyHalveUpDoubles(final double[] buf, final int start, /** * Compression algorithm used to merge higher levels. + * *

      Here is what we do for each level:

      *
      • If it does not need to be compacted, then simply copy it over.
        • *
      • Otherwise, it does need to be compacted, so... diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java index 59cda435f..fbe9dbb36 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java @@ -278,6 +278,7 @@ public final void merge(final KllSketch other) { /** * {@inheritDoc} + * *

        The parameter k will not change.

        */ @Override diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java index 50cadeb3e..69045f78c 100644 --- a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java @@ -312,6 +312,7 @@ private static void randomlyHalveUpFloats(final float[] buf, final int start, fi /** * Compression algorithm used to merge higher levels. + * *

        Here is what we do for each level:

        *
        • If it does not need to be compacted, then simply copy it over.
          • *
        • Otherwise, it does need to be compacted, so... diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java index 48781631d..b993f9998 100644 --- a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java @@ -278,6 +278,7 @@ public final void merge(final KllSketch other) { /** * {@inheritDoc} + * *

          The parameter k will not change.

          */ @Override diff --git a/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java b/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java index 00b8c7d35..dd309ba3a 100644 --- a/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java @@ -346,6 +346,7 @@ static void updateItem(final KllItemsSketch itmSk, final T item, final lo /** * Compression algorithm used to merge higher levels. + * *

          Here is what we do for each level:

          *
          • If it does not need to be compacted, then simply copy it over.
            • *
          • Otherwise, it does need to be compacted, so... diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java b/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java index ec67b55d4..04fe2cc08 100644 --- a/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java @@ -312,6 +312,7 @@ private static void randomlyHalveUpLongs(final long[] buf, final int start, fina /** * Compression algorithm used to merge higher levels. + * *

            Here is what we do for each level:

            *
            • If it does not need to be compacted, then simply copy it over.
              • *
            • Otherwise, it does need to be compacted, so... diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java index 0f6fa7d89..f5688ad70 100644 --- a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java @@ -278,6 +278,7 @@ public final void merge(final KllSketch other) { /** * {@inheritDoc} + * *

              The parameter k will not change.

              */ @Override diff --git a/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java index 49e7c5fbb..2fc399f9b 100644 --- a/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java @@ -506,6 +506,7 @@ public QuantilesDoublesSketchIterator iterator() { /** * {@inheritDoc} + * *

              The parameter k will not change.

              */ @Override diff --git a/src/main/java/org/apache/datasketches/quantiles/package-info.java b/src/main/java/org/apache/datasketches/quantiles/package-info.java index c44e2e316..8767757b7 100644 --- a/src/main/java/org/apache/datasketches/quantiles/package-info.java +++ b/src/main/java/org/apache/datasketches/quantiles/package-info.java @@ -18,9 +18,8 @@ */ /** - *

              The quantiles package contains stochastic streaming algorithms that enable single-pass + * The quantiles package contains stochastic streaming algorithms that enable single-pass * analysis of the distribution of a stream of quantiles. - *

              * * @see org.apache.datasketches.quantiles.DoublesSketch * @see org.apache.datasketches.quantiles.ItemsSketch diff --git a/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java index 98616661f..1427f6279 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java @@ -38,7 +38,7 @@ public interface DoublesSortedView extends SortedView { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + *
              *

              The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.

              @@ -55,7 +55,7 @@ public interface DoublesSortedView extends SortedView { *
            * *

            It is not recommended to include either the minimum or maximum items of the input stream.

            - * + * * @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -100,7 +100,7 @@ default double[] getCDF(double[] splitPoints, QuantileSearchCriteria searchCrit) * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + *
            *

            Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.

            * @@ -124,7 +124,7 @@ default double[] getCDF(double[] splitPoints, QuantileSearchCriteria searchCrit) *
          * *

          It is not recommended to include either the minimum or maximum items of the input stream.

          - * + * * @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedView.java index 0a0c54b5a..eec699d94 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedView.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedView.java @@ -38,7 +38,7 @@ public interface FloatsSortedView extends SortedView { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + *
          *

          The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.

          @@ -55,7 +55,7 @@ public interface FloatsSortedView extends SortedView { *
        * *

        It is not recommended to include either the minimum or maximum items of the input stream.

        - * + * * @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -100,7 +100,7 @@ default double[] getCDF(float[] splitPoints, QuantileSearchCriteria searchCrit) * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + *
        *

        Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.

        * @@ -124,7 +124,7 @@ default double[] getCDF(float[] splitPoints, QuantileSearchCriteria searchCrit) *
      * *

      It is not recommended to include either the minimum or maximum items of the input stream.

      - * + * * @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/quantilescommon/GenericSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/GenericSortedView.java index 0d27ce78c..f0dc81151 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/GenericSortedView.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/GenericSortedView.java @@ -47,7 +47,7 @@ public interface GenericSortedView extends PartitioningFeature, SketchPar * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + *
      *

      The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.

      @@ -64,7 +64,7 @@ public interface GenericSortedView extends PartitioningFeature, SketchPar *
    * *

    It is not recommended to include either the minimum or maximum items of the input stream.

    - * + * * @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -116,7 +116,7 @@ default double[] getCDF(final T[] splitPoints, final QuantileSearchCriteria sear * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + *
    *

    Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.

    * @@ -140,7 +140,7 @@ default double[] getCDF(final T[] splitPoints, final QuantileSearchCriteria sear *
* *

It is not recommended to include either the minimum or maximum items of the input stream.

- * + * * @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java index 4823edd1d..e7e3521c7 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java @@ -38,7 +38,7 @@ public interface LongsSortedView extends SortedView { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + *
*

The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.

@@ -55,7 +55,7 @@ public interface LongsSortedView extends SortedView { * * *

It is not recommended to include either the minimum or maximum items of the input stream.

- * + *
* @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -100,7 +100,7 @@ default double[] getCDF(long[] splitPoints, QuantileSearchCriteria searchCrit) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + *
*

Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.

* @@ -124,7 +124,7 @@ default double[] getCDF(long[] splitPoints, QuantileSearchCriteria searchCrit) { * * *

It is not recommended to include either the minimum or maximum items of the input stream.

- * + *
* @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java index 3dc0651a7..a082fc27a 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java @@ -20,12 +20,12 @@ package org.apache.datasketches.quantilescommon; /** - *

This is a stochastic streaming sketch that enables near-real time analysis of the + * This is a stochastic streaming sketch that enables near-real time analysis of the * approximate distribution of items from a very large stream in a single pass, requiring only * that the items are comparable. * The analysis is obtained using the getQuantile() function or the * inverse functions getRank(), getPMF() (the Probability Mass Function), and getCDF() - * (the Cumulative Distribution Function).

+ * (the Cumulative Distribution Function). * *

Given an input stream of N items, the natural rank of any specific * item is defined as its index (1 to N) in the hypothetical sorted stream of all diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesDoublesAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesDoublesAPI.java index e8e5310f5..8c4f6620f 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesDoublesAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesDoublesAPI.java @@ -50,7 +50,7 @@ default double[] getCDF(double[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + *

*

The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.

@@ -67,7 +67,7 @@ default double[] getCDF(double[] splitPoints) { * * *

It is not recommended to include either the minimum or maximum items of the input stream.

- * + *
* @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -113,7 +113,7 @@ default double[] getPMF(double[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + *
*

Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.

* @@ -137,7 +137,7 @@ default double[] getPMF(double[] splitPoints) { * * *

It is not recommended to include either the minimum or maximum items of the input stream.

- * + *
* @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesFloatsAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesFloatsAPI.java index 986780444..8b8a91bdd 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesFloatsAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesFloatsAPI.java @@ -49,7 +49,7 @@ default double[] getCDF(float[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + *
*

The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.

@@ -66,7 +66,7 @@ default double[] getCDF(float[] splitPoints) { * * *

It is not recommended to include either the minimum or maximum items of the input stream.

- * + *
* @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -112,7 +112,7 @@ default double[] getPMF(float[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + *
*

Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.

* @@ -136,7 +136,7 @@ default double[] getPMF(float[] splitPoints) { * * *

It is not recommended to include either the minimum or maximum items of the input stream.

- * + *
* @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java index 459e58cdd..bc0881282 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java @@ -53,7 +53,7 @@ default double[] getCDF(T[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + *
*

The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.

@@ -70,7 +70,7 @@ default double[] getCDF(T[] splitPoints) { * * *

It is not recommended to include either the minimum or maximum items of the input stream.

- * + *
* @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -132,7 +132,7 @@ default double[] getPMF(T[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + *
*

Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.

* @@ -156,7 +156,7 @@ default double[] getPMF(T[] splitPoints) { * * *

It is not recommended to include either the minimum or maximum items of the input stream.

- * + *
* @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java index 2b542a68f..fb1ca5817 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java @@ -50,7 +50,7 @@ default double[] getCDF(long[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + *
*

The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.

@@ -67,7 +67,7 @@ default double[] getCDF(long[] splitPoints) { * * *

It is not recommended to include either the minimum or maximum items of the input stream.

- * + *
* @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -113,7 +113,7 @@ default double[] getPMF(long[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + *
*

Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.

* @@ -137,7 +137,7 @@ default double[] getPMF(long[] splitPoints) { * * *

It is not recommended to include either the minimum or maximum items of the input stream.

- * + *
* @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java index 05d4d3ce2..feeba7739 100644 --- a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java +++ b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java @@ -172,6 +172,7 @@ public boolean isReadOnly() { /** * {@inheritDoc} + * *

The parameters k, highRankAccuracy, and reqDebug will not change.

*/ @Override diff --git a/src/main/java/org/apache/datasketches/req/ReqSerDe.java b/src/main/java/org/apache/datasketches/req/ReqSerDe.java index 52b1371a9..952749deb 100644 --- a/src/main/java/org/apache/datasketches/req/ReqSerDe.java +++ b/src/main/java/org/apache/datasketches/req/ReqSerDe.java @@ -110,7 +110,8 @@ * 0 || (empty)| 0 | K | Flags |FamID=17| SerVer | PreInts = 2 | * * 
-   * 
Flags:

+   *
+   * Flags:
    * Bit 0 : Endianness, reserved
    * Bit 1 : ReadOnly, reserved
    * Bit 2 : Empty
diff --git a/src/main/java/org/apache/datasketches/sampling/ReservoirLongsUnion.java b/src/main/java/org/apache/datasketches/sampling/ReservoirLongsUnion.java
index c3ef33957..b58317a9a 100644
--- a/src/main/java/org/apache/datasketches/sampling/ReservoirLongsUnion.java
+++ b/src/main/java/org/apache/datasketches/sampling/ReservoirLongsUnion.java
@@ -132,10 +132,9 @@ public static ReservoirLongsUnion heapify(final Memory srcMem) {
 
   /**
    * Union the given sketch.
-   * 

-   * This method can be repeatedly called. If the given sketch is null it is interpreted as an empty
-   * sketch.
-   * 

+   *
+   * 
This method can be repeatedly called. If the given sketch is null it is interpreted as an empty
+   * sketch.

    *
    * @param sketchIn The incoming sketch.
    */
diff --git a/src/main/java/org/apache/datasketches/sampling/ReservoirSize.java b/src/main/java/org/apache/datasketches/sampling/ReservoirSize.java
index e6f361955..e12d31aa9 100644
--- a/src/main/java/org/apache/datasketches/sampling/ReservoirSize.java
+++ b/src/main/java/org/apache/datasketches/sampling/ReservoirSize.java
@@ -25,6 +25,7 @@
 /**
  * This class provides a compact representation of reservoir size by encoding it into a
  * fixed-point 16-bit value.
+ *
  * 
The value itself is a fractional power of 2, with 5 bits of exponent and 11 bits of
  * mantissa. The exponent allows a choice of anywhere from 0-30, and there are 2048 possible
  * reservoir size values within each octave. Because reservoir size must be an integer, this
diff --git a/src/main/java/org/apache/datasketches/sampling/package-info.java b/src/main/java/org/apache/datasketches/sampling/package-info.java
index edfaa20a8..bbe446914 100644
--- a/src/main/java/org/apache/datasketches/sampling/package-info.java
+++ b/src/main/java/org/apache/datasketches/sampling/package-info.java
@@ -18,8 +18,8 @@
  */
 
 /**
- * 
This package is dedicated to streaming algorithms that enable fixed size, uniform sampling of
- * weighted and unweighted items from a stream.

+ * This package is dedicated to streaming algorithms that enable fixed size, uniform sampling of
+ * weighted and unweighted items from a stream.
  *
  * 
These sketches are mergeable and can be serialized and deserialized to/from a compact
  * form.

diff --git a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java
index 562be982c..e1d9262e6 100644
--- a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java
+++ b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java
@@ -37,6 +37,7 @@
 /**
  * This class defines the preamble data structure and provides basic utilities for some of the key
  * fields.
+ *
  * 
The intent of the design of this class was to isolate the detailed knowledge of the bit and
  * byte layout of the serialized form of the sketches derived from the Sketch class into one place.
  * This allows the possibility of the introduction of different serialization
@@ -126,7 +127,7 @@
  *  3   ||----------------------Start of Hash Table of longs---------------------------------|
  *
* - *

Union objects require 32 bytes of preamble plus a non-compact array of longs representing a + *

Union objects require 32 bytes of preamble plus a non-compact array of longs representing a * hash table.

* * 
diff --git a/src/main/java/org/apache/datasketches/theta/UpdateSketch.java b/src/main/java/org/apache/datasketches/theta/UpdateSketch.java
index 882c5e2e9..cb6854b02 100644
--- a/src/main/java/org/apache/datasketches/theta/UpdateSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/UpdateSketch.java
@@ -343,6 +343,7 @@ public UpdateReturnState update(final long[] data) {
 
   /**
    * All potential updates converge here.
+   *
    * 
Don't ever call this unless you really know what you are doing!

    *
    * @param hash the given input hash value.  A hash of zero or Long.MAX_VALUE is ignored.
diff --git a/src/main/java/org/apache/datasketches/tuple/Union.java b/src/main/java/org/apache/datasketches/tuple/Union.java
index 653312fa0..acefa2ab5 100644
--- a/src/main/java/org/apache/datasketches/tuple/Union.java
+++ b/src/main/java/org/apache/datasketches/tuple/Union.java
@@ -100,8 +100,7 @@ public CompactSketch union(final Sketch tupleSketch,
   /**
    * Performs a stateful union of the internal set with the given tupleSketch.
    * @param tupleSketch input tuple sketch to merge with the internal set.
-   *
-   * 
Nulls and empty sketches are ignored.

+   * Nulls and empty sketches are ignored.
    */
   public void union(final Sketch tupleSketch) {
     if (tupleSketch == null || tupleSketch.isEmpty()) { return; }
diff --git a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java
index e7abae0d4..a54c11afc 100644
--- a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java
+++ b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java
@@ -41,24 +41,28 @@ public enum Mode {
 
     /**
      * The aggregation mode is the summation function.
+     *
      * 
New retained value = previous retained value + incoming value

      */
     Sum,
 
     /**
      * The aggregation mode is the minimum function.
+     *
      * 
New retained value = min(previous retained value, incoming value)

      */
     Min,
 
     /**
      * The aggregation mode is the maximum function.
+     *
      * 
New retained value = max(previous retained value, incoming value)

      */
     Max,
 
     /**
      * The aggregation mode is always one.
+     *
      * 
New retained value = 1.0

      */
     AlwaysOne
diff --git a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java
index 4c04fa2c2..72695355e 100644
--- a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java
+++ b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java
@@ -41,24 +41,28 @@ public enum Mode {
 
     /**
      * The aggregation mode is the summation function.
+     *
      * 
New retained value = previous retained value + incoming value

      */
     Sum,
 
     /**
      * The aggregation mode is the minimum function.
+     *
      * 
New retained value = min(previous retained value, incoming value)

      */
     Min,
 
     /**
      * The aggregation mode is the maximum function.
+     *
      * 
New retained value = max(previous retained value, incoming value)

      */
     Max,
 
     /**
      * The aggregation mode is always one.
+     *
      * 
New retained value = 1

      */
     AlwaysOne
diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java
index 91d4eade4..52f827149 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java
@@ -30,6 +30,7 @@
 
 /**
  * Direct Compact Sketch of type ArrayOfDoubles.
+ *
  * 
This implementation uses data in a given Memory that is owned and managed by the caller.
  * This Memory can be off-heap, which if managed properly will greatly reduce the need for
  * the JVM to perform garbage collection.

diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesIntersection.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesIntersection.java
index 3dd019d74..7c1b1bf07 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesIntersection.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesIntersection.java
@@ -23,6 +23,7 @@
 
 /**
  * Direct Intersection operation for tuple sketches of type ArrayOfDoubles.
+ *
  * 
This implementation uses data in a given Memory that is owned and managed by the caller.
  * This Memory can be off-heap, which if managed properly will greatly reduce the need for
  * the JVM to perform garbage collection.

@@ -43,7 +44,7 @@ final class DirectArrayOfDoublesIntersection extends ArrayOfDoublesIntersection
   }
 
   @Override
-  protected ArrayOfDoublesQuickSelectSketch createSketch(final int nomEntries, final int numValues, 
+  protected ArrayOfDoublesQuickSelectSketch createSketch(final int nomEntries, final int numValues,
       final long seed) {
     return new DirectArrayOfDoublesQuickSelectSketch(nomEntries, 0, 1f, numValues, seed, mem_);
   }
diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java
index 1b4e86904..ae1aa3dc0 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java
@@ -33,6 +33,7 @@
 
 /**
  * Direct QuickSelect tuple sketch of type ArrayOfDoubles.
+ *
  * 
This implementation uses data in a given Memory that is owned and managed by the caller.
  * This Memory can be off-heap, which if managed properly will greatly reduce the need for
  * the JVM to perform garbage collection.

diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesSketchIterator.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesSketchIterator.java
index 15503fc04..dcdab1313 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesSketchIterator.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesSketchIterator.java
@@ -23,6 +23,7 @@
 
 /**
  * Iterator over the off-heap, Direct tuple sketch of type ArrayOfDoubles (compact or hash table).
+ *
  * 
This implementation uses data in a given Memory that is owned and managed by the caller.
  * This Memory can be off-heap, which if managed properly will greatly reduce the need for
  * the JVM to perform garbage collection.

diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnion.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnion.java
index 00310f534..734019632 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnion.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnion.java
@@ -27,6 +27,7 @@
 
 /**
  * Direct Union operation for tuple sketches of type ArrayOfDoubles.
+ *
  * 
This implementation uses data in a given Memory that is owned and managed by the caller.
  * This Memory can be off-heap, which if managed properly will greatly reduce the need for
  * the JVM to perform garbage collection.

diff --git a/tools/SketchesCheckstyle.xml b/tools/SketchesCheckstyle.xml
index 873a878a0..8d4b40a16 100644
--- a/tools/SketchesCheckstyle.xml
+++ b/tools/SketchesCheckstyle.xml
@@ -230,7 +230,7 @@ under the License.
       
     
     
-    
+    
 
     
       

From c57687bfe52bf4196516698e01564d7e36ffe6b3 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Wed, 11 Dec 2024 13:58:06 -0800
Subject: [PATCH 28/88] Update README.md

---
 README.md | 98 ++++++++++---------------------------------------------
 1 file changed, 17 insertions(+), 81 deletions(-)

diff --git a/README.md b/README.md
index 80517d304..547df3e9d 100644
--- a/README.md
+++ b/README.md
@@ -25,10 +25,10 @@
 # Apache® DataSketches™ Core Java Library Component
 This is the core Java component of the DataSketches library.  It contains all of the sketching algorithms and can be accessed directly from user applications. 
 
-This component is also a dependency of other components of the library that create adaptors for target systems, such as the [Apache Pig adaptor](https://github.com/apache/datasketches-pig) and the [Apache Hive adaptor](https://github.com/apache/datasketches-hive).
+This component is also a dependency of other components of the library that create adaptors for target systems, such as the [Apache Pig adaptor](https://github.com/apache/datasketches-pig), the [Apache Hive adaptor](https://github.com/apache/datasketches-hive), and others.
 
 Note that we have a parallel core component for C++ and Python implementations of the same sketch algorithms, 
-[datasketches-cpp](https://github.com/apache/datasketches-cpp).
+[datasketches-cpp](https://github.com/apache/datasketches-cpp) and [datasketches-python](https://github.com/apache/datasketches-python)
 
 Please visit the main [DataSketches website](https://datasketches.apache.org) for more information. 
 
@@ -39,30 +39,29 @@ If you are interested in making contributions to this site please see our [Commu
 ## Maven Build Instructions
 __NOTE:__ This component accesses resource files for testing. As a result, the directory elements of the full absolute path of the target installation directory must qualify as Java identifiers. In other words, the directory elements must not have any space characters (or non-Java identifier characters) in any of the path elements. This is required by the Oracle Java Specification in order to ensure location-independent access to resources: [See Oracle Location-Independent Access to Resources](https://docs.oracle.com/javase/8/docs/technotes/guides/lang/resources.html)
 
-### A JDK8 with Hotspot or JDK11 with Hotspot is required to compile
-This component depends on the [datasketches-memory](https://github.com/apache/datasketches-memory) component, 
-and, as a result, must be compiled with one of the above JDKs. 
-If your application only relies on the APIs of this component no special JVM arguments are required.
-However, if your application also directly relies on the APIs of the *datasketches-memory* component, 
-you may need additional JVM arguments.
-Please refer to the [datasketches-memory README](https://github.com/apache/datasketches-memory/blob/master/README.md) for details.
+### JDK17 is required to compile
+This component depends on the [datasketches-memory-4.1.X](https://github.com/apache/datasketches-memory/tree/4.1.X) component, 
+and, as a result, must be compiled with JDK17 and this dependency:
 
-If your application uses Maven, you can also use the *pom.xml* of this component as an example of how to automatically
-configure the JVM arguments for compilation and testing based on the version of the JDK.
+```
+
+  org.apache.datasketches
+  datasketches-memory
+  4.1.0
+
+```
+
+If your application only relies on the APIs of datasketches-java no special JVM arguments are required.
+However, if your application also directly relies on the APIs of the *datasketches-memory* component, 
+you may need the additional JVM argument **--enable-preview**.
 
 ### Recommended Build Tool
 This DataSketches component is structured as a Maven project and Maven is the recommended Build Tool.
 
-There are two types of tests: normal unit tests and tests run by the strict profile.  
-
 To run normal unit tests:
 
     $ mvn clean test
 
-To run the strict profile tests (only supported in Java 8):
-
-    $ mvn clean test -P strict
-
 To install jars built from the downloaded source:
 
     $ mvn clean install -DskipTests=true
@@ -80,74 +79,11 @@ This will create the following jars:
 #### Run-time
 There is one run-time dependency: 
 
-* org.apache.datasketches : datasketches-memory
+* [datasketches-memory-4.1.X](https://github.com/apache/datasketches-memory/tree/4.1.X)
 
 #### Testing
 See the pom.xml file for test dependencies.
 
-## Special Build / Test Instructions for Eclipse
-
-Building and running tests using JDK 8 should not be a problem. 
-
-However, with JDK 9+, and Eclipse versions up to and including 4.22.0 (2021-12), Eclipse fails to translate the required JPMS JVM arguments specified in the POM compiler or surefire plugins into the *.classpath* file, causing illegal reflection access errors 
-[eclipse-m2e/m2e-core Bug 543631](https://github.com/eclipse-m2e/m2e-core/issues/129).
-
-There are two ways to fix this:
-
-#### Method 1: Manually update *.classpath* file:
-Open the *.classpath* file in a text editor and find the following *classpathentry* element (this assumes JDK11, change to suit):
-
-```
-	
-		
-			
-			
-		
-	
-```
-Then edit it as follows:
-
-```
-	
-		
-			
-			
-			
-			
-		
-	
-```
-Finally, *refresh*.
-
-#### Method 2: Manually update *Module Dependencies*
-
-In Eclipse, open the project *Properties / Java Build Path / Module Dependencies ...*
-
-* Select *java.base*
-* Select *Configured details*
-* Select *Expose Package...*
-    * Enter *Package* = java.nio
-    * Enter *Target module* = ALL-UNNAMED
-    * Select button: *opens*
-    * Hit *OK*
-* Select *Expose Package...*
-    * Enter *Package* = jdk.internal.misc
-    * Enter *Target module* = ALL-UNNAMED
-    * Select button: *exports*
-    * Hit *OK*
-* Select *Expose Package...*
-    * Enter *Package* = jdk.internal.ref
-    * Enter *Target module* = ALL-UNNAMED
-    * Select button: *exports*
-    * Hit *OK*
-* Select *Expose Package...*
-    * Enter *Package* = sun.nio.ch
-    * Enter *Target module* = ALL-UNNAMED
-    * Select button: *opens*
-    * Hit *OK*
-
-**NOTE:** If you execute *Maven/Update Project...* from Eclipse with the option *Update project configuration from pom.xml* checked, all of the above will be erased, and you will have to redo it.
-
 ## Known Issues
 
 #### SpotBugs

From 5fc852f5149402ee098e857bfdb9ad39ada53e85 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Wed, 11 Dec 2024 14:02:13 -0800
Subject: [PATCH 29/88] Update pom in preparation to create main based on 17.

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index ccff0122d..4c5532e54 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,7 +33,7 @@ under the License.
 
   org.apache.datasketches
   datasketches-java
-  7.0.0
+  7.0.0-SNAPSHOT
   jar
 
   ${project.artifactId}

From d0575f7176e0d6c7c016caec01d0aeeb6488aaa2 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Tue, 17 Dec 2024 12:18:05 -0800
Subject: [PATCH 30/88] Update move_to_17_using_FFM with GHA workflow changes.

---
 ...cpp_files.yml => auto-check_cpp_files.yml} | 32 +++++++++++++------
 .github/workflows/auto-jdk-matrix.yml         | 22 +++++++------
 ...nalysis.yml => manual-codeql-analysis.yml} | 11 ++-----
 .../{javadoc.yml => manual-javadoc.yml}       |  6 ++--
 4 files changed, 38 insertions(+), 33 deletions(-)
 rename .github/workflows/{check_cpp_files.yml => auto-check_cpp_files.yml} (59%)
 rename .github/workflows/{codeql-analysis.yml => manual-codeql-analysis.yml} (84%)
 rename .github/workflows/{javadoc.yml => manual-javadoc.yml} (85%)

diff --git a/.github/workflows/check_cpp_files.yml b/.github/workflows/auto-check_cpp_files.yml
similarity index 59%
rename from .github/workflows/check_cpp_files.yml
rename to .github/workflows/auto-check_cpp_files.yml
index 59ae5824a..b9e05ae97 100644
--- a/.github/workflows/check_cpp_files.yml
+++ b/.github/workflows/auto-check_cpp_files.yml
@@ -2,8 +2,7 @@ name: Serialization Compatibility Test
 
 on:
   push:
-    branches:
-      - master
+    branches: [ master, main ]
   workflow_dispatch:
 
 jobs:
@@ -12,21 +11,34 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Checkout C++
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+      
+	  - name: Checkout C++
+        uses: actions/checkout@v4
         with:
           repository: apache/datasketches-cpp
           path: cpp
-      - name: Configure C++ build
+      
+	  - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+      
+	  - name: Configure C++ build
         run: cd cpp/build && cmake .. -DGENERATE=true
-      - name: Build C++ unit tests
+      
+	  - name: Build C++ unit tests
         run: cd cpp && cmake --build build --config Release
-      - name: Run C++ tests
+      
+	  - name: Run C++ tests
         run: cd cpp && cmake --build build --config Release --target test
-      - name: Make dir
+      
+	  - name: Make dir
         run: mkdir -p serialization_test_data/cpp_generated_files
+      
       - name: Copy files
         run: cp cpp/build/*/test/*_cpp.sk serialization_test_data/cpp_generated_files
-      - name: Run Java tests
+      
+	  - name: Run Java tests
         run: mvn test -P check-cpp-files
diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml
index 6e280f123..f8220bbe2 100644
--- a/.github/workflows/auto-jdk-matrix.yml
+++ b/.github/workflows/auto-jdk-matrix.yml
@@ -7,7 +7,7 @@ on:
     workflow_dispatch:
 
 env:
-    MAVEN_OPTS: -Xmx1g -Xms1g
+    MAVEN_OPTS: -Xmx4g -Xms1g
 
 jobs:
     build:
@@ -16,9 +16,11 @@ jobs:
 
         strategy:
           fail-fast: false
+          matrix:
+              jdk: [ 17 ]
 
         env:
-          JDK_VERSION: 17
+          JDK_VERSION: ${{ matrix.jdk }}
 
         steps:
         - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -30,14 +32,6 @@ jobs:
           run: >
              cat .github/workflows/auto-jdk-matrix.yml
 
-        - name: Setup Java
-          uses: actions/setup-java@v4
-          with:
-              distribution: 'temurin'
-              java-package: jdk
-              architecture: x64
-              java-version: 17        
-        
         - name: Cache local Maven repository
           uses: actions/cache@v4
           with:
@@ -45,6 +39,14 @@ jobs:
               key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
               restore-keys: build-${{ runner.os }}-maven-
 
+        - name: Install Matrix JDK
+          uses: actions/setup-java@v4
+          with:
+              java-version: ${{ matrix.jdk }}
+              distribution: 'temurin'
+              java-package: jdk
+              architecture: x64 
+
         - name: Echo Java Version
           run: >
               java -version
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/manual-codeql-analysis.yml
similarity index 84%
rename from .github/workflows/codeql-analysis.yml
rename to .github/workflows/manual-codeql-analysis.yml
index 376d69404..1b94d13a3 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/manual-codeql-analysis.yml
@@ -1,13 +1,6 @@
 name: "CodeQL"
 
 on:
-#  push:
-#    branches: [ 'master' ]
-#  pull_request:
-    # The branches below must be a subset of the branches above
-#    branches: [ 'master' ]
-#  schedule:
-#    - cron: '10 17 * * 4'
   workflow_dispatch:
 
 jobs:
@@ -48,8 +41,8 @@ jobs:
     - name: Autobuild
       uses: github/codeql-action/autobuild@v4
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    #   Command-line programs to run using the OS shell.
+    #   See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
     #   If the Autobuild fails above, remove it and uncomment the following three lines. 
     #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
diff --git a/.github/workflows/javadoc.yml b/.github/workflows/manual-javadoc.yml
similarity index 85%
rename from .github/workflows/javadoc.yml
rename to .github/workflows/manual-javadoc.yml
index 04d70e25a..5f6ac6aec 100644
--- a/.github/workflows/javadoc.yml
+++ b/.github/workflows/manual-javadoc.yml
@@ -1,8 +1,6 @@
 name: JavaDoc
 
 on:
-  push:
-    branches: [ master, main ]
   workflow_dispatch:
 
 jobs:
@@ -24,7 +22,7 @@ jobs:
 
       - name: Print Current workflow
         run: >
-         cat .github/workflows/javadoc.yml
+         cat .github/workflows/manual-javadoc.yml
 
       - name: Generate JavaDoc
         run: mvn javadoc:javadoc
@@ -33,6 +31,6 @@ jobs:
         uses: JamesIves/github-pages-deploy-action@v4.6.8
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          folder: target/reports/apidocs
+          folder: target/site/apidocs
           target-folder: docs/${{ github.ref_name }}
           branch: gh-pages

From 8752c6aaa349e1afa817abd1ef9cd922062b06c3 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Tue, 17 Dec 2024 12:19:38 -0800
Subject: [PATCH 31/88] update move_to_17_using_FFM with 6.1.X

---
 .../frequencies/PreambleUtil.java             |  16 +-
 .../quantiles/DoublesByteArrayImpl.java       |  24 +-
 .../datasketches/tdigest/TDigestDouble.java   |  48 +-
 .../bloomfilter/DirectBitArrayTest.java       |   2 +-
 .../hll/DirectCouponListTest.java             |   3 -
 .../datasketches/hll/PreambleUtilTest.java    |   4 -
 .../kll/KllCrossLanguageTest.java             |  10 +
 .../KllDirectCompactDoublesSketchTest.java    |  14 +-
 .../KllDirectDoublesSketchIteratorTest.java   |   2 +-
 .../kll/KllDirectDoublesSketchTest.java       |   8 +-
 .../kll/KllDoublesSketchSerDeTest.java        |   2 +-
 .../kll/KllDoublesSketchTest.java             |  14 +-
 .../datasketches/kll/KllItemsSketchTest.java  |   2 +-
 .../datasketches/kll/KllMiscDoublesTest.java  |   4 +-
 .../datasketches/kll/KllMiscItemsTest.java    |  20 +-
 .../quantiles/DebugUnionTest.java             |   1 -
 .../DirectQuantilesMemoryRequestTest.java     |   1 -
 .../quantiles/DoublesSketchTest.java          |   1 -
 .../tdigest/TDigestDoubleTest.java            |  10 +
 .../datasketches/theta/CompactSketchTest.java |   1 -
 ...ConcurrentDirectQuickSelectSketchTest.java |   1 -
 .../ConcurrentHeapQuickSelectSketchTest.java  |   2 +-
 .../theta/DirectQuickSelectSketchTest.java    |   9 +-
 .../theta/HeapifyWrapSerVer1and2Test.java     | 833 +++++++++---------
 .../datasketches/theta/SketchesTest.java      |   3 +-
 .../theta/ThetaSketchCrossLanguageTest.java   |  10 +
 .../datasketches/theta/UnionImplTest.java     |   3 +-
 tools/SketchesCheckstyle.xml                  |   2 +-
 28 files changed, 554 insertions(+), 496 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java b/src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java
index b5db6025c..1021a89ce 100644
--- a/src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java
+++ b/src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java
@@ -32,22 +32,16 @@
  * This class defines the preamble data structure and provides basic utilities for some of the key
  * fields.
  *
- * 

- * The intent of the design of this class was to isolate the detailed knowledge of the bit and byte
+ * 
The intent of the design of this class was to isolate the detailed knowledge of the bit and byte
  * layout of the serialized form of the sketches derived from the Sketch class into one place. This
  * allows the possibility of the introduction of different serialization schemes with minimal impact
- * on the rest of the library.
- * 

+ * on the rest of the library.

  *
- * 

- * MAP: Low significance bytes of this long data structure are on the right. However, the
+ * 
MAP: Low significance bytes of this long data structure are on the right. However, the
  * multi-byte integers (int and long) are stored in native byte order. The byte
- * values are treated as unsigned.
- * 

+ * values are treated as unsigned.

  *
- * 

- * An empty FrequentItems only requires 8 bytes. All others require 32 bytes of preamble.
- * 

+ * 
An empty FrequentItems only requires 8 bytes. All others require 32 bytes of preamble.

  *
  * 
  *
- * The raw BitArray bits, if non-empty start at byte 24.
+ * The raw BitArray bits, if non-empty, start at byte 32.
  */
 
   /**

From feef1fafb0e42a5726b63090cf5fb687f63e0be8 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Fri, 21 Feb 2025 11:54:31 -0800
Subject: [PATCH 74/88] Fix typo in .asf.yaml

---
 .asf.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.asf.yaml b/.asf.yaml
index f7627bc08..28ead840c 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -12,7 +12,7 @@ github:
 
       required_pull_request_reviews:
         dismiss_stale_reviews: false
-        equired_approving_review_count: 1
+        ecquired_approving_review_count: 1
       
       # squash or rebase must be allowed in the repo for this setting to be set to true.
       required_linear_history: false

From 305ec24cbec58e5b8b07166644c162439c69e5d7 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Fri, 21 Feb 2025 11:57:06 -0800
Subject: [PATCH 75/88] Fix typo fix

---
 .asf.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.asf.yaml b/.asf.yaml
index 28ead840c..a149bf396 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -12,7 +12,7 @@ github:
 
       required_pull_request_reviews:
         dismiss_stale_reviews: false
-        ecquired_approving_review_count: 1
+        required_approving_review_count: 1
       
       # squash or rebase must be allowed in the repo for this setting to be set to true.
       required_linear_history: false

From 94c9f6e927147aac419b139660f0da38a5d1706f Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Sat, 22 Feb 2025 10:36:05 -0800
Subject: [PATCH 76/88] Last change to pom.

Update ds-memory to 6.0.0
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 1d829adc4..d858e9cb1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -83,7 +83,7 @@ under the License.
   
 
   
-    6.1.0-SNAPSHOT
+    6.0.0
 
     
     7.10.2

From 588d5859cfabb12169fa487aba4c1693884b12a8 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Sat, 22 Feb 2025 10:44:30 -0800
Subject: [PATCH 77/88] Update workflows to java21

---
 .github/workflows/auto-jdk-matrix.yml | 2 +-
 .github/workflows/auto-os-matrix.yml  | 2 +-
 .github/workflows/check_cpp_files.yml | 2 +-
 .github/workflows/codeql-analysis.yml | 2 +-
 .github/workflows/javadoc.yml         | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml
index 8c20cc742..0afbaf065 100644
--- a/.github/workflows/auto-jdk-matrix.yml
+++ b/.github/workflows/auto-jdk-matrix.yml
@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [ 17 ]
+        jdk: [ 21 ]
 
     env:
       JDK_VERSION: ${{ matrix.jdk }}
diff --git a/.github/workflows/auto-os-matrix.yml b/.github/workflows/auto-os-matrix.yml
index 65be62f98..83ecc1ffe 100644
--- a/.github/workflows/auto-os-matrix.yml
+++ b/.github/workflows/auto-os-matrix.yml
@@ -21,7 +21,7 @@ jobs:
       fail-fast: false
 
       matrix:
-        jdk: [ 17 ]
+        jdk: [ 21 ]
         os: [ windows-latest, ubuntu-latest, macos-latest ]
         include:
           - os: windows-latest
diff --git a/.github/workflows/check_cpp_files.yml b/.github/workflows/check_cpp_files.yml
index feac7a55a..243eda985 100644
--- a/.github/workflows/check_cpp_files.yml
+++ b/.github/workflows/check_cpp_files.yml
@@ -27,7 +27,7 @@ jobs:
       - name: Setup Java
         uses: actions/setup-java@v4
         with:
-          java-version: '17'
+          java-version: '21'
           distribution: 'temurin'
 
       - name: Configure C++ build
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 1102d3149..bb42fe345 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -35,7 +35,7 @@ jobs:
         with:
           distribution: 'temurin'
           cache: 'maven'
-          java-version: '17'
+          java-version: '21'
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v3
diff --git a/.github/workflows/javadoc.yml b/.github/workflows/javadoc.yml
index bd5903836..4862d64e4 100644
--- a/.github/workflows/javadoc.yml
+++ b/.github/workflows/javadoc.yml
@@ -16,7 +16,7 @@ jobs:
       - name: Setup Java
         uses: actions/setup-java@v4
         with:
-          java-version: '17'
+          java-version: '21'
           distribution: 'temurin'
 
       - name: Echo Java Version

From e1662cbdfc4da9fe249fbb8b830983a87b0af4b9 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Mon, 24 Feb 2025 18:58:39 -0800
Subject: [PATCH 78/88] Made some minor changes.

Some of the //TODOs were left over from Kevin's C code, so I left the
comment but removed the TODO.  Java can't do "INLINE CODE".

This includes a new, very detailed test of the HLL growth transitions
and sizes.
---
 .../datasketches/cpc/CpcCompression.java      |   4 +-
 .../datasketches/hll/DirectCouponHashSet.java |  10 +-
 .../datasketches/hll/HeapAuxHashMap.java      |   1 -
 .../datasketches/cpc/CpcCompressionTest.java  |   4 +-
 .../datasketches/hll/CouponListTest.java      |   2 +-
 .../hll/SizeAndModeTransitions.java           | 103 ++++++++++++++++++
 .../DirectQuantilesMemoryRequestTest.java     |   5 +-
 7 files changed, 119 insertions(+), 10 deletions(-)
 create mode 100644 src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java

diff --git a/src/main/java/org/apache/datasketches/cpc/CpcCompression.java b/src/main/java/org/apache/datasketches/cpc/CpcCompression.java
index f811d8db1..aa73b94e7 100644
--- a/src/main/java/org/apache/datasketches/cpc/CpcCompression.java
+++ b/src/main/java/org/apache/datasketches/cpc/CpcCompression.java
@@ -288,7 +288,7 @@ static long lowLevelCompressPairs(
       final long golombLo = yDelta & golombLoMask; //long for bitBuf
       final long golombHi = yDelta >>> numBaseBits; //cannot exceed 2^26
 
-      //TODO Inline WriteUnary
+      // Inline WriteUnary
       ptrArr[NEXT_WORD_IDX] = nextWordIndex;
       ptrArr[BIT_BUF] = bitBuf;
       ptrArr[BUF_BITS] = bufBits;
@@ -372,7 +372,7 @@ static void lowLevelUncompressPairs(
       bitBuf >>>= codeWordLength;
       bufBits -= codeWordLength;
 
-      //TODO Inline ReadUnary
+      // Inline ReadUnary
       ptrArr[NEXT_WORD_IDX] = nextWordIndex;
       ptrArr[BIT_BUF] = bitBuf;
       ptrArr[BUF_BITS] = bufBits;
diff --git a/src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java b/src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java
index 52ddf60ba..683c8be96 100644
--- a/src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java
+++ b/src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java
@@ -35,6 +35,7 @@
 import static org.apache.datasketches.hll.PreambleUtil.insertLgArr;
 
 import org.apache.datasketches.common.SketchesArgumentException;
+import org.apache.datasketches.common.SketchesException;
 import org.apache.datasketches.common.SketchesStateException;
 import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.memory.WritableMemory;
@@ -105,13 +106,14 @@ private boolean checkGrowOrPromote() {
       if (lgCouponArrInts == (getLgConfigK() - 3)) {
         return true; // promote
       }
-      //TODO if direct, ask for more memory
       insertLgArr(wmem, ++lgCouponArrInts);
       growHashSet(wmem, lgCouponArrInts);
     }
     return false;
   }
 
+  //This could fail if the user has undersized the given WritableMemory
+  //  and not used the public methods for sizing the Memory.  See exception.
   private static final void growHashSet(final WritableMemory wmem, final int tgtLgCouponArrSize) {
     final int tgtArrSize = 1 << tgtLgCouponArrSize;
     final int[] tgtCouponIntArr = new int[tgtArrSize];
@@ -128,7 +130,11 @@ private static final void growHashSet(final WritableMemory wmem, final int tgtLg
       }
     }
     wmem.clear(HASH_SET_INT_ARR_START, tgtArrSize << 2);
-    wmem.putIntArray(HASH_SET_INT_ARR_START, tgtCouponIntArr, 0, tgtArrSize);
+    try { wmem.putIntArray(HASH_SET_INT_ARR_START, tgtCouponIntArr, 0, tgtArrSize); }
+    catch (final IndexOutOfBoundsException e) {
+      throw new SketchesException(
+          "The WritableMemory is undersized. Use the public methods for properly sizing Memory.", e);
+    }
   }
 
   //Searches the Coupon hash table (embedded in Memory) for an empty slot
diff --git a/src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java b/src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java
index 374dbd624..5724b563c 100644
--- a/src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java
+++ b/src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java
@@ -205,7 +205,6 @@ else if (slotNo == (arrVal & configKmask)) { //Compares only on slotNo
   private void checkGrow() {
     if ((RESIZE_DENOM * auxCount) > (RESIZE_NUMER * auxIntArr.length)) {
       growAuxSpace();
-      //TODO if direct, ask for more memory
     }
   }
 
diff --git a/src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java b/src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java
index 9a33aa71f..c5ec12399 100644
--- a/src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java
+++ b/src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java
@@ -56,7 +56,7 @@ public void checkWriteReadUnary() {
 
     for (int i = 0; i < 100; i++) {
 
-      //TODO Inline WriteUnary
+      // Inline WriteUnary
       ptrArr[NEXT_WORD_IDX] = nextWordIndex;
       ptrArr[BIT_BUF] = bitBuf;
       ptrArr[BUF_BITS] = bufBits;
@@ -92,7 +92,7 @@ public void checkWriteReadUnary() {
 
     for (int i = 0; i < 100; i++) {
 
-      //TODO Inline ReadUnary
+      // Inline ReadUnary
       ptrArr[NEXT_WORD_IDX] = nextWordIndex;
       ptrArr[BIT_BUF] = bitBuf;
       ptrArr[BUF_BITS] = bufBits;
diff --git a/src/test/java/org/apache/datasketches/hll/CouponListTest.java b/src/test/java/org/apache/datasketches/hll/CouponListTest.java
index dbd5bfb0b..774e2c304 100644
--- a/src/test/java/org/apache/datasketches/hll/CouponListTest.java
+++ b/src/test/java/org/apache/datasketches/hll/CouponListTest.java
@@ -42,7 +42,7 @@ private static void checkIterator(boolean direct, int lgK, int n) {
     TgtHllType tgtHllType = TgtHllType.HLL_4;
     int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
     WritableMemory wmem = WritableMemory.allocate(bytes);
-    HllSketch sk = (direct) ? new HllSketch(lgK, tgtHllType, wmem) : new HllSketch(lgK);
+    HllSketch sk = (direct) ? new HllSketch(lgK, tgtHllType, wmem) : new HllSketch(lgK, tgtHllType);
     for (int i = 0; i < n; i++) { sk.update(i); }
     String store = direct ? "Memory" : "Heap";
     println("CurMode: " + sk.getCurMode().toString() + "; Store: " + store);
diff --git a/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java b/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java
new file mode 100644
index 000000000..c18ce395d
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll;
+
+import static org.testng.Assert.assertEquals;
+
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+public class SizeAndModeTransitions {
+  static String hfmt = "%6s %7s %4s %10s %5s %10s %10s %10s %10s %10s %14s\n";
+  static String dfmt = "%6s %7s %4d %,10d %5s %,10d %,10d %,10d %,10d %,10d %,14.3f\n";
+  static String[] hdr = {"Type", "Store", "LgK", "N", "Mode", "ActCBytes", "CmpBytes", "ActUBytes", "UpdBytes", "MaxBytes", "Estimate"};
+
+  @Test
+  public void checkHLL8Heap() {
+    TgtHllType tgtHllType = TgtHllType.HLL_8;
+    boolean direct = false;
+    int lgK = 10;
+    int N = 97;
+    printf(hfmt, (Object[]) hdr);
+    int maxBytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
+    WritableMemory wmem = null;;
+    HllSketch sk;
+    if (direct) {
+      wmem = WritableMemory.allocate(maxBytes);
+      sk = new HllSketch(lgK, tgtHllType, wmem);
+    } else {
+      sk = new HllSketch(lgK, tgtHllType);
+    }
+    String type = tgtHllType.toString();
+    String store = direct ? "Memory" : "Heap";
+    for (int i = 1; i <= N; i++) {
+      sk.update(i);
+      if (i == 7)  { checkAtN(sk, tgtHllType, store, lgK, i, "LIST",  36,   40, 1064); }
+      if (i == 8)  { checkAtN(sk, tgtHllType, store, lgK, i, "SET",   44,  140, 1064); }
+      if (i == 24) { checkAtN(sk, tgtHllType, store, lgK, i, "SET",  108,  140, 1064); }
+      if (i == 25) { checkAtN(sk, tgtHllType, store, lgK, i, "SET",  112,  268, 1064); }
+      if (i == 48) { checkAtN(sk, tgtHllType, store, lgK, i, "SET",  204,  268, 1064); }
+      if (i == 49) { checkAtN(sk, tgtHllType, store, lgK, i, "SET",  208,  524, 1064); }
+      if (i == 96) { checkAtN(sk, tgtHllType, store, lgK, i, "SET",  396,  524, 1064); }
+      if (i == 97) { checkAtN(sk, tgtHllType, store, lgK, i, "HLL", 1064, 1064, 1064); }
+    }
+    println("");
+  }
+
+  private static void checkAtN(HllSketch sk, TgtHllType tgtHllType, String store, int lgK, int n, String trueMode,
+      int cmpTrueBytes, int updTrueBytes, int maxTrueBytes) {
+    int maxBytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
+    String type = tgtHllType.toString();
+    int cmpBytes = sk.getCompactSerializationBytes();
+    int updBytes = sk.getUpdatableSerializationBytes();
+    byte[] actCBytes = sk.toCompactByteArray();
+    byte[] actUBytes = sk.toUpdatableByteArray();
+    String mode = sk.getCurMode().toString();
+    double est = sk.getEstimate();
+    printf(dfmt, type, store, lgK, n, mode,  actCBytes.length, cmpBytes, actUBytes.length, updBytes, maxBytes, est);
+    assertEquals(mode, trueMode);
+    assertEquals(cmpBytes, actCBytes.length);
+    assertEquals(cmpBytes, cmpTrueBytes);
+    assertEquals(updBytes, actUBytes.length);
+    assertEquals(updBytes, updTrueBytes);
+    assertEquals(maxBytes, maxTrueBytes);
+  }
+
+  @Test
+  public void printlnTest() {
+    println("PRINTING: "+this.getClass().getName());
+  }
+
+  /**
+   * @param s value to print
+   */
+  static void println(String s) {
+    System.out.println(s); //disable here
+  }
+
+  /**
+   * @param fmt format
+   * @param args arguments
+   */
+  static void printf(String fmt, Object...args) {
+    System.out.printf(fmt, args); //disable here
+  }
+
+}
diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
index 1c8819981..b61171ca1 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
@@ -125,7 +125,8 @@ public void checkUpdatableStorageBytes() {
     usk1.update(1.0);
     byte[] uarr = usk1.toByteArray();
     println("Actual Storage Bytes " + uarr.length);
-    assertEquals(initBytes, uarr.length); //64
+    assertEquals(initBytes, uarr.length);
+    assertEquals(initBytes, 64);
   }
 
 
@@ -151,7 +152,7 @@ public void checkGrowFromWrappedEmptySketch() {
     assertEquals(usk2.getN(), 1);
     WritableMemory mem2 = usk2.getMemory();
     assertFalse(wmem.isSameResource(mem2));
-    assertFalse(mem2.isDirect()); //should now be on-heap //TODO
+    assertFalse(mem2.isDirect()); //should now be on-heap
 
     final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3);
     assertEquals(mem2.getCapacity(), expectedSize);

From 9212cbd9c83bc5e892f749fdf6d1f172963de328 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Mon, 24 Feb 2025 19:00:36 -0800
Subject: [PATCH 79/88] Suppressed some print statements.

---
 .../org/apache/datasketches/hll/SizeAndModeTransitions.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java b/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java
index c18ce395d..ad7a4e509 100644
--- a/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java
+++ b/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java
@@ -89,7 +89,7 @@ public void printlnTest() {
    * @param s value to print
    */
   static void println(String s) {
-    System.out.println(s); //disable here
+    //System.out.println(s); //disable here
   }
 
   /**
@@ -97,7 +97,7 @@ static void println(String s) {
    * @param args arguments
    */
   static void printf(String fmt, Object...args) {
-    System.out.printf(fmt, args); //disable here
+    //System.out.printf(fmt, args); //disable here
   }
 
 }

From 9e5b51c8f426a06f0b03da38438f7398defcd50a Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Mon, 24 Feb 2025 22:28:13 -0800
Subject: [PATCH 80/88] touch-up changes mostly to clean up code contributed by
 others.

---
 .../datasketches/theta/CompactSketch.java     | 51 ++++++++++++++++++-
 .../theta/DirectCompactCompressedSketch.java  |  4 +-
 .../datasketches/theta/IntersectionImpl.java  |  6 +--
 .../org/apache/datasketches/theta/Sketch.java |  2 +-
 .../theta/WrappedCompactSketch.java           |  2 +-
 .../hll/SizeAndModeTransitions.java           |  1 -
 6 files changed, 56 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/theta/CompactSketch.java b/src/main/java/org/apache/datasketches/theta/CompactSketch.java
index b64314168..630b2e135 100644
--- a/src/main/java/org/apache/datasketches/theta/CompactSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/CompactSketch.java
@@ -228,14 +228,61 @@ else if (serVer == 2) {
         "Corrupted: Serialization Version " + serVer + " not recognized.");
   }
 
+  /**
+   * Wrap takes the sketch image in the given Memory and refers to it directly.
+   * There is no data copying onto the java heap.
+   * The wrap operation enables fast read-only merging and access to all the public read-only API.
+   *
+   * 
Only "Direct" Serialization Version 3 (i.e, OpenSource) sketches that have
+   * been explicitly stored as direct sketches can be wrapped.
+   * Wrapping earlier serial version sketches will result in a heapify operation.
+   * These early versions were never designed to "wrap".

+   *
+   * 
Wrapping any subclass of this class that is empty or contains only a single item will
+   * result in heapified forms of empty and single item sketch respectively.
+   * This is actually faster and consumes less overall memory.

+   *
+   * 
This method checks if the DEFAULT_UPDATE_SEED was used to create the source Memory image.
+   * Note that SerialVersion 1 sketches cannot be checked as they don't have a seedHash field,
+   * so the resulting heapified CompactSketch will be given the hash of DEFAULT_UPDATE_SEED.

+   *
+   * @param bytes a byte array image of a Sketch that was created using the DEFAULT_UPDATE_SEED.
+   * See Memory
+   *
+   * @return a CompactSketch backed by the given Memory except as above.
+   */
   public static CompactSketch wrap(final byte[] bytes) {
     return wrap(bytes, ThetaUtil.DEFAULT_UPDATE_SEED, false);
   }
-  
+
+  /**
+   * Wrap takes the sketch image in the given Memory and refers to it directly.
+   * There is no data copying onto the java heap.
+   * The wrap operation enables fast read-only merging and access to all the public read-only API.
+   *
+   * 
Only "Direct" Serialization Version 3 (i.e, OpenSource) sketches that have
+   * been explicitly stored as direct sketches can be wrapped.
+   * Wrapping earlier serial version sketches will result in a heapify operation.
+   * These early versions were never designed to "wrap".

+   *
+   * 
Wrapping any subclass of this class that is empty or contains only a single item will
+   * result in heapified forms of empty and single item sketch respectively.
+   * This is actually faster and consumes less overall memory.

+   *
+   * 
This method checks if the given expectedSeed was used to create the source Memory image.
+   * Note that SerialVersion 1 sketches cannot be checked as they don't have a seedHash field,
+   * so the resulting heapified CompactSketch will be given the hash of the expectedSeed.

+   *
+   * @param bytes a byte array image of a Sketch that was created using the given expectedSeed.
+   * See Memory
+   * @param expectedSeed the seed used to validate the given Memory image.
+   * See Update Hash Seed.
+   * @return a CompactSketch backed by the given Memory except as above.
+   */
   public static CompactSketch wrap(final byte[] bytes, final long expectedSeed) {
     return wrap(bytes, expectedSeed, true);
   }
-  
+
   private static CompactSketch wrap(final byte[] bytes, final long seed, final boolean enforceSeed) {
     final int serVer = bytes[PreambleUtil.SER_VER_BYTE];
     final int familyId = bytes[PreambleUtil.FAMILY_BYTE];
diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java
index 6c83add87..64c0fafd4 100644
--- a/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java
@@ -82,7 +82,7 @@ public int getCurrentBytes() {
 
   private static final int START_PACKED_DATA_EXACT_MODE = 8;
   private static final int START_PACKED_DATA_ESTIMATION_MODE = 16;
-  
+
   @Override
   public int getRetainedEntries(final boolean valid) { //compact is always valid
     // number of entries is stored using variable length encoding
@@ -132,7 +132,7 @@ long[] getCache() {
     final int numEntries = getRetainedEntries();
     final long[] cache = new long[numEntries];
     int i = 0;
-    HashIterator it = iterator();
+    final HashIterator it = iterator();
     while (it.next()) {
       cache[i++] = it.get();
     }
diff --git a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java
index b1be73c74..772480fea 100644
--- a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java
+++ b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java
@@ -504,13 +504,13 @@ private void moveDataToTgt(final long[] arr, final int count) {
   }
 
   private void moveDataToTgt(final Sketch sketch) {
-    int count = sketch.getRetainedEntries();
+    final int count = sketch.getRetainedEntries();
     int tmpCnt = 0;
     if (wmem_ != null) { //Off Heap puts directly into mem
       final int preBytes = CONST_PREAMBLE_LONGS << 3;
       final int lgArrLongs = lgArrLongs_;
       final long thetaLong = thetaLong_;
-      HashIterator it = sketch.iterator();
+      final HashIterator it = sketch.iterator();
       while (it.next()) {
         final long hash = it.get();
         if (continueCondition(thetaLong, hash)) { continue; }
@@ -518,7 +518,7 @@ private void moveDataToTgt(final Sketch sketch) {
         tmpCnt++;
       }
     } else { //On Heap. Assumes HT exists and is large enough
-      HashIterator it = sketch.iterator();
+      final HashIterator it = sketch.iterator();
       while (it.next()) {
         final long hash = it.get();
         if (continueCondition(thetaLong_, hash)) { continue; }
diff --git a/src/main/java/org/apache/datasketches/theta/Sketch.java b/src/main/java/org/apache/datasketches/theta/Sketch.java
index 6583e2dbf..cb202a189 100644
--- a/src/main/java/org/apache/datasketches/theta/Sketch.java
+++ b/src/main/java/org/apache/datasketches/theta/Sketch.java
@@ -472,7 +472,7 @@ public String toString(final boolean sketchSummary, final boolean dataDetail, fi
       final int w = width > 0 ? width : 8; // default is 8 wide
       if (curCount > 0) {
         sb.append("### SKETCH DATA DETAIL");
-        HashIterator it = iterator();
+        final HashIterator it = iterator();
         int j = 0;
         while (it.next()) {
           final long h = it.get();
diff --git a/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java b/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java
index b1073159b..519857d21 100644
--- a/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java
@@ -136,7 +136,7 @@ public byte[] toByteArray() {
   long[] getCache() {
     final long[] cache = new long[getRetainedEntries()];
     int i = 0;
-    HashIterator it = iterator();
+    final HashIterator it = iterator();
     while (it.next()) {
       cache[i++] = it.get();
     }
diff --git a/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java b/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java
index ad7a4e509..c9ffa1e28 100644
--- a/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java
+++ b/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java
@@ -45,7 +45,6 @@ public void checkHLL8Heap() {
     } else {
       sk = new HllSketch(lgK, tgtHllType);
     }
-    String type = tgtHllType.toString();
     String store = direct ? "Memory" : "Heap";
     for (int i = 1; i <= N; i++) {
       sk.update(i);

From f776ff0522b881498436030acdde883e5fe08eed Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Sat, 15 Mar 2025 10:07:51 -0700
Subject: [PATCH 81/88] clean up public methods w/o javadocs

---
 pom.xml                                            |  7 ++++---
 .../org/apache/datasketches/common/Family.java     |  2 +-
 .../apache/datasketches/cpc/MergingValidation.java |  4 ++--
 .../datasketches/cpc/QuickMergingValidation.java   |  4 ++--
 .../datasketches/cpc/StreamingValidation.java      |  4 ++--
 .../java/org/apache/datasketches/cpc/TestUtil.java |  1 +
 .../java/org/apache/datasketches/fdt/Group.java    | 14 +++++++++-----
 .../datasketches/frequencies/ItemsSketch.java      |  7 +++++++
 .../datasketches/frequencies/LongsSketch.java      |  9 ++++++++-
 .../quantiles/UpdateDoublesSketch.java             |  2 ++
 .../quantilescommon/QuantilesGenericAPI.java       |  1 +
 .../datasketches/sampling/SampleSubsetSummary.java |  8 ++++++--
 .../apache/datasketches/tdigest/TDigestDouble.java |  5 +++++
 .../java/org/apache/datasketches/theta/AnotB.java  |  5 +++++
 .../apache/datasketches/theta/SetOperation.java    |  3 +++
 .../datasketches/thetacommon/HashOperations.java   |  2 ++
 .../datasketches/tuple/DeserializeResult.java      |  4 +++-
 .../java/org/apache/datasketches/tuple/Sketch.java |  1 +
 .../org/apache/datasketches/tuple/Sketches.java    |  1 +
 .../apache/datasketches/tuple/SummaryFactory.java  |  1 +
 .../java/org/apache/datasketches/tuple/Util.java   |  6 ++++--
 .../datasketches/tuple/adouble/DoubleSketch.java   |  1 +
 .../datasketches/tuple/adouble/DoubleSummary.java  |  1 +
 .../tuple/adouble/DoubleSummaryDeserializer.java   |  1 +
 .../tuple/aninteger/IntegerSketch.java             |  1 +
 .../tuple/aninteger/IntegerSummary.java            |  1 +
 .../aninteger/IntegerSummaryDeserializer.java      |  1 +
 .../tuple/arrayofdoubles/ArrayOfDoublesSketch.java | 12 ++++++++++--
 .../tuple/arrayofdoubles/ArrayOfDoublesUnion.java  |  2 ++
 .../tuple/strings/ArrayOfStringsSketch.java        |  1 +
 .../tuple/strings/ArrayOfStringsSummary.java       |  2 ++
 .../strings/ArrayOfStringsSummaryDeserializer.java |  1 +
 .../strings/ArrayOfStringsSummaryFactory.java      |  1 +
 .../ArrayOfStringsSummarySetOperations.java        |  1 +
 34 files changed, 94 insertions(+), 23 deletions(-)

diff --git a/pom.xml b/pom.xml
index d858e9cb1..8aaa8174e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -237,9 +237,10 @@ under the License.
           
             true
             public
-            
-              ${jvm-ffm-flag}
-            
+            all,-missing
+            
+              ${jvm-ffm-flag}
+            
           
           
             
diff --git a/src/main/java/org/apache/datasketches/common/Family.java b/src/main/java/org/apache/datasketches/common/Family.java
index 225248997..829a95960 100644
--- a/src/main/java/org/apache/datasketches/common/Family.java
+++ b/src/main/java/org/apache/datasketches/common/Family.java
@@ -194,7 +194,7 @@ public int getID() {
   }
 
   /**
-   *
+   * Checks the given Family ID
    * @param id the given id, a value < 128.
    */
   public void checkFamilyID(final int id) {
diff --git a/src/main/java/org/apache/datasketches/cpc/MergingValidation.java b/src/main/java/org/apache/datasketches/cpc/MergingValidation.java
index dae3670ea..213e65d30 100644
--- a/src/main/java/org/apache/datasketches/cpc/MergingValidation.java
+++ b/src/main/java/org/apache/datasketches/cpc/MergingValidation.java
@@ -56,7 +56,7 @@ public class MergingValidation {
   private PrintWriter printWriter;
 
   /**
-   *
+   * Constructor
    * @param lgMinK lgMinK
    * @param lgMaxK lgMaxK
    * @param lgMulK lgMulK
@@ -79,7 +79,7 @@ public MergingValidation(final int lgMinK, final int lgMaxK, final int lgMulK, f
   }
 
   /**
-   *
+   * Start the merging validation process
    */
   public void start() {
     printf(hfmt, (Object[]) hStrArr); //print header
diff --git a/src/main/java/org/apache/datasketches/cpc/QuickMergingValidation.java b/src/main/java/org/apache/datasketches/cpc/QuickMergingValidation.java
index 389f3e4dd..e2bb8605f 100644
--- a/src/main/java/org/apache/datasketches/cpc/QuickMergingValidation.java
+++ b/src/main/java/org/apache/datasketches/cpc/QuickMergingValidation.java
@@ -54,7 +54,7 @@ public class QuickMergingValidation {
   private PrintWriter printWriter;
 
   /**
-   *
+   * Constructor
    * @param lgMinK lgMinK
    * @param lgMaxK lgMaxK
    * @param incLgK incLgK
@@ -73,7 +73,7 @@ public QuickMergingValidation(final int lgMinK, final int lgMaxK, final int incL
   }
 
   /**
-   *
+   * Start the Quick Merging Validation process
    */
   public void start() {
     printf(hfmt, (Object[]) hStrArr); //print header
diff --git a/src/main/java/org/apache/datasketches/cpc/StreamingValidation.java b/src/main/java/org/apache/datasketches/cpc/StreamingValidation.java
index 97dc4a9b2..aab6e5e6f 100644
--- a/src/main/java/org/apache/datasketches/cpc/StreamingValidation.java
+++ b/src/main/java/org/apache/datasketches/cpc/StreamingValidation.java
@@ -57,7 +57,7 @@ public class StreamingValidation {
   private BitMatrix matrix = null;
 
   /**
-   *
+   * Constructor
    * @param lgMinK lgMinK
    * @param lgMaxK lgMaxK
    * @param trials trials
@@ -78,7 +78,7 @@ public StreamingValidation(final int lgMinK, final int lgMaxK, final int trials,
   }
 
   /**
-   *
+   * Start the streaming validation process
    */
   public void start() {
     printf(hfmt, (Object[]) hStrArr);
diff --git a/src/main/java/org/apache/datasketches/cpc/TestUtil.java b/src/main/java/org/apache/datasketches/cpc/TestUtil.java
index 0a96db768..442e3ffe6 100644
--- a/src/main/java/org/apache/datasketches/cpc/TestUtil.java
+++ b/src/main/java/org/apache/datasketches/cpc/TestUtil.java
@@ -25,6 +25,7 @@
 import static org.apache.datasketches.cpc.RuntimeAsserts.rtAssertEquals;
 
 /**
+ * Utility methods for Test
  * @author Lee Rhodes
  */
 public class TestUtil {
diff --git a/src/main/java/org/apache/datasketches/fdt/Group.java b/src/main/java/org/apache/datasketches/fdt/Group.java
index 67017699d..e22ee2398 100644
--- a/src/main/java/org/apache/datasketches/fdt/Group.java
+++ b/src/main/java/org/apache/datasketches/fdt/Group.java
@@ -66,41 +66,49 @@ public Group init(final String priKey, final int count, final double estimate, f
   }
 
   /**
-   * @return priKey of type T
+   * Gets the primary key of type String
+   * @return priKey of type String
    */
   public String getPrimaryKey() { return priKey; }
 
   /**
+   * Returns the count
    * @return the count
    */
   public int getCount() { return count; }
 
   /**
+   * Returns the estimate
    * @return the estimate
    */
   public double getEstimate() { return est; }
 
   /**
+   * Returns the upper bound
    * @return the upper bound
    */
   public double getUpperBound() { return ub; }
 
   /**
+   * Returns the lower bound
    * @return the lower bound
    */
   public double getLowerBound() { return lb; }
 
   /**
+   * Returns the fraction for this group
    * @return the fraction for this group
    */
   public double getFraction() { return fraction; }
 
   /**
+   * Returns the RSE
    * @return the RSE
    */
   public double getRse() { return rse; }
 
   /**
+   * Returns the descriptive header
    * @return the descriptive header
    */
   public String getHeader() {
@@ -112,9 +120,6 @@ public String toString() {
     return String.format(fmt, count, est, ub, lb, fraction, rse, priKey);
   }
 
-  /**
-   * @param that The Group to compare to
-   */
   @Override
   public int compareTo(final Group that) {
     return that.count - count; //decreasing
@@ -133,4 +138,3 @@ public int hashCode() {
   }
 
 }
-
diff --git a/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java b/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java
index b2648b943..042612e42 100644
--- a/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java
+++ b/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java
@@ -397,6 +397,7 @@ public Row[] getFrequentItems(final ErrorType errorType) {
   }
 
   /**
+   * Returns an upper bound on the maximum error of getEstimate(item) for any item.
    * @return An upper bound on the maximum error of getEstimate(item) for any item.
    * This is equivalent to the maximum distance between the upper bound and the lower bound
    * for any item.
@@ -415,6 +416,7 @@ public int getMaximumMapCapacity() {
   }
 
   /**
+   * Returns the number of active items in the sketch.
    * @return the number of active items in the sketch.
    */
   public int getNumActiveItems() {
@@ -624,26 +626,31 @@ public static class Row implements Comparable> {
     }
 
     /**
+     * Returns an item of type T
      * @return item of type T
      */
     public T getItem() { return item; }
 
     /**
+     * Returns the estimate
      * @return the estimate
      */
     public long getEstimate() { return est; }
 
     /**
+     * Returns the upper bound
      * @return the upper bound
      */
     public long getUpperBound() { return ub; }
 
     /**
+     * Returns the lower bound
      * @return return the lower bound
      */
     public long getLowerBound() { return lb; }
 
     /**
+     * Returns the descriptive row header
      * @return the descriptive row header
      */
     public static String getRowHeader() {
diff --git a/src/main/java/org/apache/datasketches/frequencies/LongsSketch.java b/src/main/java/org/apache/datasketches/frequencies/LongsSketch.java
index 7d1990b62..1fdc52894 100644
--- a/src/main/java/org/apache/datasketches/frequencies/LongsSketch.java
+++ b/src/main/java/org/apache/datasketches/frequencies/LongsSketch.java
@@ -439,6 +439,7 @@ public Row[] getFrequentItems(final ErrorType errorType) {
   }
 
   /**
+   * Returns an upper bound on the maximum error of getEstimate(item) for any item.
    * @return An upper bound on the maximum error of getEstimate(item) for any item.
    * This is equivalent to the maximum distance between the upper bound and the lower bound
    * for any item.
@@ -457,6 +458,7 @@ public int getMaximumMapCapacity() {
   }
 
   /**
+   * Returns the number of active items in the sketch.
    * @return the number of active items in the sketch.
    */
   public int getNumActiveItems() {
@@ -693,26 +695,31 @@ public static class Row implements Comparable {
     }
 
     /**
-     * @return item of type T
+     * Returns item of type long
+     * @return item of type long
      */
     public long getItem() { return item; }
 
     /**
+     * Returns the estimate
      * @return the estimate
      */
     public long getEstimate() { return est; }
 
     /**
+     * Returns the upper bound
      * @return the upper bound
      */
     public long getUpperBound() { return ub; }
 
     /**
+     * Returns the lower bound
      * @return return the lower bound
      */
     public long getLowerBound() { return lb; }
 
     /**
+     * Returns the descriptive row header
      * @return the descriptive row header
      */
     public static String getRowHeader() {
diff --git a/src/main/java/org/apache/datasketches/quantiles/UpdateDoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/UpdateDoublesSketch.java
index f56c3b352..f7191402d 100644
--- a/src/main/java/org/apache/datasketches/quantiles/UpdateDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/quantiles/UpdateDoublesSketch.java
@@ -23,6 +23,7 @@
 import org.apache.datasketches.memory.WritableMemory;
 
 /**
+ * Extends DoubleSketch
  * @author Jon Malkin
  */
 public abstract class UpdateDoublesSketch extends DoublesSketch {
@@ -61,6 +62,7 @@ public static UpdateDoublesSketch heapify(final Memory srcMem) {
   }
 
   /**
+   * Returns a CompactDoublesSketch of this class
    * @return a CompactDoublesSketch of this class
    */
   public CompactDoublesSketch compact() {
diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java
index bc0881282..c6a05ffa1 100644
--- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java
+++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java
@@ -78,6 +78,7 @@ default double[] getCDF(T[] splitPoints) {
   double[] getCDF(T[] splitPoints, QuantileSearchCriteria searchCrit);
 
   /**
+   * Returns the sketch item class
    * @return the sketch item class
    */
   Class getClassOfT();
diff --git a/src/main/java/org/apache/datasketches/sampling/SampleSubsetSummary.java b/src/main/java/org/apache/datasketches/sampling/SampleSubsetSummary.java
index b7758b148..1504542f7 100644
--- a/src/main/java/org/apache/datasketches/sampling/SampleSubsetSummary.java
+++ b/src/main/java/org/apache/datasketches/sampling/SampleSubsetSummary.java
@@ -41,13 +41,15 @@ public class SampleSubsetSummary {
   }
 
   /**
-   * @return the Lower Bound
+   * Returns the lower bound
+   * @return the lower bound
    */
   public double getLowerBound() {
     return lowerBound;
   }
 
   /**
+   * Returns the total sketch weight
    * @return the total sketch weight
    */
   public double getTotalSketchWeight() {
@@ -55,13 +57,15 @@ public double getTotalSketchWeight() {
   }
 
   /**
-   * @return the Upper Bound
+   * Returns the upper bound
+   * @return the upper bound
    */
   public double getUpperBound() {
     return upperBound;
   }
 
   /**
+   * Returns the unique count estimate
    * @return the unique count estimate
    */
   public double getEstimate() {
diff --git a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java
index 951bd7244..d44a439d1 100644
--- a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java
+++ b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java
@@ -86,6 +86,7 @@ public TDigestDouble(final short k) {
   }
 
   /**
+   * Returns parameter k (compression) that was used to configure this TDigest
    * @return parameter k (compression) that was used to configure this TDigest
    */
   public short getK() {
@@ -137,6 +138,7 @@ private void compress() {
   }
 
   /**
+   * Returns true if TDigest has not seen any data
    * @return true if TDigest has not seen any data
    */
   public boolean isEmpty() {
@@ -144,6 +146,7 @@ public boolean isEmpty() {
   }
 
   /**
+   * Returns minimum value seen by TDigest
    * @return minimum value seen by TDigest
    */
   public double getMinValue() {
@@ -152,6 +155,7 @@ public double getMinValue() {
   }
 
   /**
+   * Returns maximum value seen by TDigest
    * @return maximum value seen by TDigest
    */
   public double getMaxValue() {
@@ -160,6 +164,7 @@ public double getMaxValue() {
   }
 
   /**
+   * Returns total weight
    * @return total weight
    */
   public long getTotalWeight() {
diff --git a/src/main/java/org/apache/datasketches/theta/AnotB.java b/src/main/java/org/apache/datasketches/theta/AnotB.java
index 322e66ef7..d9ab37416 100644
--- a/src/main/java/org/apache/datasketches/theta/AnotB.java
+++ b/src/main/java/org/apache/datasketches/theta/AnotB.java
@@ -56,6 +56,11 @@
  */
 public abstract class AnotB extends SetOperation {
 
+  /**
+   * Constructor
+   */
+  AnotB() {}
+
   @Override
   public Family getFamily() {
     return Family.A_NOT_B;
diff --git a/src/main/java/org/apache/datasketches/theta/SetOperation.java b/src/main/java/org/apache/datasketches/theta/SetOperation.java
index b89ca9703..c198dceb7 100644
--- a/src/main/java/org/apache/datasketches/theta/SetOperation.java
+++ b/src/main/java/org/apache/datasketches/theta/SetOperation.java
@@ -39,6 +39,9 @@
 public abstract class SetOperation implements MemoryStatus {
   static final int CONST_PREAMBLE_LONGS = 3;
 
+  /**
+   * Constructor
+   */
   SetOperation() {}
 
   /**
diff --git a/src/main/java/org/apache/datasketches/thetacommon/HashOperations.java b/src/main/java/org/apache/datasketches/thetacommon/HashOperations.java
index 848bb47ed..f6b22cb50 100644
--- a/src/main/java/org/apache/datasketches/thetacommon/HashOperations.java
+++ b/src/main/java/org/apache/datasketches/thetacommon/HashOperations.java
@@ -287,6 +287,7 @@ public static int hashSearchOrInsertMemory(final WritableMemory wmem, final int
   //Other related methods
 
   /**
+   * Checks that the given theta is not negative nor zero.
    * @param thetaLong must be greater than zero otherwise throws an exception.
    * See Theta Long
    */
@@ -299,6 +300,7 @@ public static void checkThetaCorruption(final long thetaLong) {
   }
 
   /**
+   * Checks that the given hash value is not negative.
    * @param hash must be greater than -1 otherwise throws an exception.
    * Note a hash of zero is normally ignored, but a negative hash is never allowed.
    */
diff --git a/src/main/java/org/apache/datasketches/tuple/DeserializeResult.java b/src/main/java/org/apache/datasketches/tuple/DeserializeResult.java
index 21adb45e2..db8ad682b 100644
--- a/src/main/java/org/apache/datasketches/tuple/DeserializeResult.java
+++ b/src/main/java/org/apache/datasketches/tuple/DeserializeResult.java
@@ -38,6 +38,7 @@ public DeserializeResult(final T object, final int size) {
   }
 
   /**
+   * Returns Deserialized object
    * @return Deserialized object
    */
   public T getObject() {
@@ -45,7 +46,8 @@ public T getObject() {
   }
 
   /**
-   * @return Size in bytes occupied by the object in the serialized form
+   * Returns size in bytes occupied by the object in the serialized form
+   * @return size in bytes occupied by the object in the serialized form
    */
   public int getSize() {
     return size;
diff --git a/src/main/java/org/apache/datasketches/tuple/Sketch.java b/src/main/java/org/apache/datasketches/tuple/Sketch.java
index dcc7a2c16..f0003bf3c 100644
--- a/src/main/java/org/apache/datasketches/tuple/Sketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/Sketch.java
@@ -145,6 +145,7 @@ public boolean isEstimationMode() {
   }
 
   /**
+   * Returns number of retained entries
    * @return number of retained entries
    */
   public abstract int getRetainedEntries();
diff --git a/src/main/java/org/apache/datasketches/tuple/Sketches.java b/src/main/java/org/apache/datasketches/tuple/Sketches.java
index b4c1eed14..fba8ab604 100644
--- a/src/main/java/org/apache/datasketches/tuple/Sketches.java
+++ b/src/main/java/org/apache/datasketches/tuple/Sketches.java
@@ -28,6 +28,7 @@
 public final class Sketches {
 
   /**
+   * Creates an empty sketch.
    * @param  Type of Summary
    * @return an empty instance of Sketch
    */
diff --git a/src/main/java/org/apache/datasketches/tuple/SummaryFactory.java b/src/main/java/org/apache/datasketches/tuple/SummaryFactory.java
index 4b20b6772..5c5648f9d 100644
--- a/src/main/java/org/apache/datasketches/tuple/SummaryFactory.java
+++ b/src/main/java/org/apache/datasketches/tuple/SummaryFactory.java
@@ -26,6 +26,7 @@
 public interface SummaryFactory {
 
   /**
+   * Returns new instance of Summary
    * @return new instance of Summary
    */
   public S newSummary();
diff --git a/src/main/java/org/apache/datasketches/tuple/Util.java b/src/main/java/org/apache/datasketches/tuple/Util.java
index 88fef2602..92193ca56 100644
--- a/src/main/java/org/apache/datasketches/tuple/Util.java
+++ b/src/main/java/org/apache/datasketches/tuple/Util.java
@@ -119,16 +119,18 @@ public static String stringConcat(final String[] strArr) {
   }
 
   /**
+   * Returns the hash of the given string
    * @param s the string to hash
-   * @return the hash of the string
+   * @return the hash of the given string
    */
   public static long stringHash(final String s) {
     return hashString(s, 0, s.length(), PRIME);
   }
 
   /**
+   * Returns the hash of the concatenated strings
    * @param strArray array of Strings
-   * @return long hash of concatenated strings.
+   * @return the hash of concatenated strings.
    */
   public static long stringArrHash(final String[] strArray) {
     final String s = stringConcat(strArray);
diff --git a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSketch.java b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSketch.java
index d78f4e8b1..1bb9edeca 100644
--- a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSketch.java
@@ -24,6 +24,7 @@
 import org.apache.datasketches.tuple.UpdatableSketch;
 
 /**
+ * Extends UpdatableSketch<Double, DoubleSummary>
  * @author Lee Rhodes
  */
 public class DoubleSketch extends UpdatableSketch {
diff --git a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java
index a54c11afc..f678fba06 100644
--- a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java
+++ b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java
@@ -125,6 +125,7 @@ public DoubleSummary copy() {
   }
 
   /**
+   * Returns current value of the DoubleSummary
    * @return current value of the DoubleSummary
    */
   public double getValue() {
diff --git a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummaryDeserializer.java b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummaryDeserializer.java
index f8fae67a7..90e01b72b 100644
--- a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummaryDeserializer.java
+++ b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummaryDeserializer.java
@@ -24,6 +24,7 @@
 import org.apache.datasketches.tuple.SummaryDeserializer;
 
 /**
+ * Implements SummaryDeserializer<DoubleSummary>
  * @author Lee Rhodes
  */
 public class DoubleSummaryDeserializer implements SummaryDeserializer {
diff --git a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSketch.java b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSketch.java
index 438879824..0bad790cc 100644
--- a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSketch.java
@@ -24,6 +24,7 @@
 import org.apache.datasketches.tuple.UpdatableSketch;
 
 /**
+ * Extends UpdatableSketch<Integer, IntegerSummary>
  * @author Lee Rhodes
  */
 public class IntegerSketch extends UpdatableSketch {
diff --git a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java
index 72695355e..a8cfdab56 100644
--- a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java
+++ b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java
@@ -125,6 +125,7 @@ public IntegerSummary copy() {
   }
 
   /**
+   * Returns the current value of the IntegerSummary
    * @return current value of the IntegerSummary
    */
   public int getValue() {
diff --git a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummaryDeserializer.java b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummaryDeserializer.java
index 0a4529e07..68d468785 100644
--- a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummaryDeserializer.java
+++ b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummaryDeserializer.java
@@ -24,6 +24,7 @@
 import org.apache.datasketches.tuple.SummaryDeserializer;
 
 /**
+ * Implements SummaryDeserializer<IntegerSummary>
  * @author Lee Rhodes
  */
 public class IntegerSummaryDeserializer implements SummaryDeserializer {
diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java
index 9453c872d..111427b7c 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java
@@ -172,6 +172,7 @@ public boolean isEmpty() {
   }
 
   /**
+   * Returns number of double values associated with each key
    * @return number of double values associated with each key
    */
   public int getNumValues() {
@@ -196,11 +197,13 @@ public double getTheta() {
   }
 
   /**
+   * Returns number of retained entries
    * @return number of retained entries
    */
   public abstract int getRetainedEntries();
 
   /**
+   * Returns the maximum number of bytes for this sketch when serialized.
    * @return the maximum number of bytes for this sketch when serialized.
    */
   public abstract int getMaxBytes();
@@ -212,11 +215,13 @@ public double getTheta() {
   public abstract int getCurrentBytes();
 
   /**
+   * Returns serialized representation of the sketch
    * @return serialized representation of the sketch
    */
   public abstract byte[] toByteArray();
 
   /**
+   * Returns array of arrays of double values in the sketch
    * @return array of arrays of double values in the sketch
    */
   public abstract double[][] getValues();
@@ -226,6 +231,7 @@ public double getTheta() {
   abstract long[] getKeys();
 
   /**
+   * Returns the value of theta as a long
    * @return the value of theta as a long
    */
   long getThetaLong() {
@@ -235,11 +241,13 @@ long getThetaLong() {
   abstract short getSeedHash();
 
   /**
-   * @return iterator over the sketch
+   * Returns an iterator over the sketch
+   * @return an iterator over the sketch
    */
   public abstract ArrayOfDoublesSketchIterator iterator();
 
   /**
+   * Returns this sketch in compact form, which is immutable.
    * @return this sketch in compact form, which is immutable.
    */
   public ArrayOfDoublesCompactSketch compact() {
@@ -247,7 +255,7 @@ public ArrayOfDoublesCompactSketch compact() {
   }
 
   /**
-   *
+   * Returns this sketch in compact form, which is immutable.
    * @param dstMem the destination WritableMemory
    * @return this sketch in compact form, which is immutable.
    */
diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnion.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnion.java
index 599418f75..9e1db0ada 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnion.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnion.java
@@ -189,6 +189,7 @@ public void reset() {
   //      ||   15   |   14   |   13   |   12   |   11    |   10   |    9   |     8              |
   //  1   ||---------------------------Union Theta Long-----------------------------------------|
   /**
+   * Returns a byte array representation of this object
    * @return a byte array representation of this object
    */
   public byte[] toByteArray() {
@@ -206,6 +207,7 @@ public byte[] toByteArray() {
   }
 
   /**
+   * Returns maximum required storage bytes given nomEntries and numValues
    * @param nomEntries Nominal number of entries. Forced to the nearest power of 2 greater than or equal to
    * given value.
    * @param numValues Number of double values to keep for each key
diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketch.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketch.java
index c66802f06..99ddeee16 100644
--- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketch.java
@@ -26,6 +26,7 @@
 import org.apache.datasketches.tuple.UpdatableSketch;
 
 /**
+ * Extends UpdatableSketch<String[], ArrayOfStringsSummary>
  * @author Lee Rhodes
  */
 public class ArrayOfStringsSketch extends UpdatableSketch {
diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummary.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummary.java
index 69ee83e8e..48b2d6d6a 100644
--- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummary.java
+++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummary.java
@@ -31,6 +31,7 @@
 import org.apache.datasketches.tuple.UpdatableSummary;
 
 /**
+ * Implements UpdatableSummary<String[]>
  * @author Lee Rhodes
  */
 public final class ArrayOfStringsSummary implements UpdatableSummary {
@@ -115,6 +116,7 @@ public boolean equals(final Object summary) {
   }
 
   /**
+   * Returns the nodes array for this summary.
    * @return the nodes array for this summary.
    */
   public String[] getValue() {
diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryDeserializer.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryDeserializer.java
index 1fb47db1a..16467f338 100644
--- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryDeserializer.java
+++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryDeserializer.java
@@ -24,6 +24,7 @@
 import org.apache.datasketches.tuple.SummaryDeserializer;
 
 /**
+ * Implements SummaryDeserializer<ArrayOfStringsSummary>
  * @author Lee Rhodes
  */
 public class ArrayOfStringsSummaryDeserializer implements SummaryDeserializer {
diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryFactory.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryFactory.java
index 8294b2bae..98fb7fb58 100644
--- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryFactory.java
+++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryFactory.java
@@ -22,6 +22,7 @@
 import org.apache.datasketches.tuple.SummaryFactory;
 
 /**
+ * Implements SummaryFactory<ArrayOfStringsSummary>
  * @author Lee Rhodes
  */
 public class ArrayOfStringsSummaryFactory implements SummaryFactory {
diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummarySetOperations.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummarySetOperations.java
index dd24eabaa..2650b84ae 100644
--- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummarySetOperations.java
+++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummarySetOperations.java
@@ -22,6 +22,7 @@
 import org.apache.datasketches.tuple.SummarySetOperations;
 
 /**
+ * Implements SummarySetOperations<ArrayOfStringsSummary>
  * @author Lee Rhodes
  */
 public class ArrayOfStringsSummarySetOperations implements SummarySetOperations {

From a56289babb6a2180b13e0c7ab3a511b83b895d1a Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Thu, 27 Mar 2025 13:59:26 -0700
Subject: [PATCH 82/88] Set lower bound of enforced Java versions to 21.

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 8aaa8174e..983abb657 100644
--- a/pom.xml
+++ b/pom.xml
@@ -190,7 +190,7 @@ under the License.
               
                 
                   
-                    ${java.version}
+                    21, 
                   
                   
                     [${maven.version},4.0.0)

From 1757d77274d2d430d60bb12ee9a30f398545368c Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Sat, 29 Mar 2025 16:14:02 -0700
Subject: [PATCH 83/88] adjust maven-enforcer-plugin java version bounds.

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 983abb657..84e032947 100644
--- a/pom.xml
+++ b/pom.xml
@@ -190,7 +190,7 @@ under the License.
               
                 
                   
-                    21, 
+                    [21,) 
                   
                   
                     [${maven.version},4.0.0)

From 11de5f2259e010da1a0fc0aefa15069c158d6eaa Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Sun, 30 Mar 2025 17:06:27 -0700
Subject: [PATCH 84/88] Update 8.0.X POM to 8.0.0 (remove -SNAPSHOT).

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 84e032947..752eb3654 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,7 +33,7 @@ under the License.
 
   org.apache.datasketches
   datasketches-java
-  8.0.0-SNAPSHOT
+  8.0.0
   jar
 
   ${project.artifactId}

From 2014087801972e43757e4ddd32de16921915b552 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Tue, 8 Apr 2025 12:40:13 -0700
Subject: [PATCH 85/88] Update the README.md file

---
 README.md | 61 ++++++++++++++++++++++++++-----------------------------
 1 file changed, 29 insertions(+), 32 deletions(-)

diff --git a/README.md b/README.md
index b6db7f89e..9022928f8 100644
--- a/README.md
+++ b/README.md
@@ -27,42 +27,49 @@ This is the core Java component of the DataSketches library.  It contains all of
 
 This component is also a dependency of other components of the library that create adaptors for target systems, such as the [Apache Pig adaptor](https://github.com/apache/datasketches-pig), the [Apache Hive adaptor](https://github.com/apache/datasketches-hive), and others.
 
-Note that we have a parallel core component for C++ and Python implementations of the same sketch algorithms, 
+Note that we have a parallel core component for C++ and Python implementations of many of the same sketch algorithms, 
 [datasketches-cpp](https://github.com/apache/datasketches-cpp) and [datasketches-python](https://github.com/apache/datasketches-python)
 
-Please visit the main [DataSketches website](https://datasketches.apache.org) for more information. 
+Please visit the main [DataSketches website](https://datasketches.apache.org) for more information.
 
 If you are interested in making contributions to this site please see our [Community](https://datasketches.apache.org/docs/Community/) page for how to contact us.
 
 ---
 
-## Maven Build Instructions
-__NOTE:__ This component accesses resource files for testing. As a result, the directory elements of the full absolute path of the target installation directory must qualify as Java identifiers. In other words, the directory elements must not have any space characters (or non-Java identifier characters) in any of the path elements. This is required by the Oracle Java Specification in order to ensure location-independent access to resources: [See Oracle Location-Independent Access to Resources](https://docs.oracle.com/javase/8/docs/technotes/guides/lang/resources.html)
+## Build & Runtime Dependencies
 
-### JDK17 is required to compile
-This component depends on the [datasketches-memory-4.1.X](https://github.com/apache/datasketches-memory/tree/4.1.X) component, 
-and, as a result, must be compiled with JDK17 and this dependency:
+### Installation Directory Path
+**NOTE:** This component accesses resource files for testing. As a result, the directory elements of the full absolute path of the target installation directory must qualify as Java identifiers. In other words, the directory elements must not have any space characters (or non-Java identifier characters) in any of the path elements. This is required by the Oracle Java Specification in order to ensure location-independent access to resources: [See Oracle Location-Independent Access to Resources](https://docs.oracle.com/javase/8/docs/technotes/guides/lang/resources.html)
 
-```
-
-  org.apache.datasketches
-  datasketches-memory
-  4.1.0
-
-```
+### OpenJDK Version 21
+An OpenJDK-compatible build of Java 21, provided by one of the Open-Source providers, such as Azul Systems, Red Hat, SAP, Eclipse Temurin, etc, is required.
+All of the testing of this release has been performed with a Eclipse Temurin build.
 
-If your application only relies on the APIs of datasketches-java no special JVM arguments are required.
-However, if your application also directly relies on the APIs of the *datasketches-memory* component, 
-you may need the additional JVM argument **--add-modules=jdk.incubator.foreign**.
+This release uses the new Java Foreign Function & Memory (FFM) features that are in "preview" in Java 21.
+As a result, the JVM flag **--enable-preview** must be set at compile and at runtime.
 
-### Recommended Build Tool
-This DataSketches component is structured as a Maven project and Maven is the recommended Build Tool.
+**NOTE:** OpenJDK versions greater than 21 do not support running Java 21 class files (Class ID 65) with preview code. The runtime JVM version must be 21.
 
-To run normal unit tests:
+**NOTE:** The Eclipse Compiler for Java (ECJ) version 21, used by default in both the Eclipse JDT IDE and the VScode IDE, will not allow compilation of preview code for Java version 21. Both Eclipse and VScode can be configured to use an OpenJDK compiler instead, but you will loose the incremental build capability of the ECJ.
+
+### DataSketches Memory 6.0.0
+
+This component depends on the [datasketches-memory-6.0.0](https://github.com/apache/datasketches-memory/tree/6.0.0) component, 
+
+## Compilation and Test using Maven
+This DataSketches component is structured as a Maven project and Maven is the recommended tool for compile and test.
+
+#### A Toolchain is required
+
+* You must have a JDK type toolchain defined in location *~/.m2/toolchains.xml* that specifies where to find a locally installed OpenJDK-compatible version 21.
+* Your default \$JAVA\_HOME compiler must be an OpenJDK compatible, specified in the toolchain, and may be a version greater than 21. Note that if your \$JAVA\_HOME is set to a Java version greater than 21, Maven will automatically use the Java 21 version specified in the toolchain instead. The included pom.xml specifies the necessary JVM flags, so no further action should be required.
+* Note that the paths specified in the toolchain must be fully qualified direct paths to the OpenJDK version locations. Using environment variables will not work.
+
+#### To run normal unit tests:
 
     $ mvn clean test
 
-To install jars built from the downloaded source:
+#### To install jars built from the downloaded source:
 
     $ mvn clean install -DskipTests=true
 
@@ -74,19 +81,9 @@ This will create the following jars:
 * datasketches-java-X.Y.Z-test-sources.jar The test source files
 * datasketches-java-X.Y.Z-javadoc.jar  The compressed Javadocs.
 
-### Dependencies
-
-#### Run-time
-There is one run-time dependency: 
-
-* [datasketches-memory-4.1.X](https://github.com/apache/datasketches-memory/tree/4.1.X)
-
-#### Testing
-See the pom.xml file for test dependencies.
-
 ## Known Issues
 
-#### SpotBugs
+### SpotBugs
 
 * Make sure you configure SpotBugs with the /tools/FindBugsExcludeFilter.xml file. Otherwise, you may get a lot of false positive or low risk issues that we have examined and eliminated with this exclusion file.
 

From 90cb79d5d3ea7b8a84fd385c6a2488094156e48b Mon Sep 17 00:00:00 2001
From: Dmitry Kryukov 
Date: Mon, 7 Apr 2025 15:12:04 +0300
Subject: [PATCH 86/88] Fixed octal number 1 - need 0.01 instead. At least
 according to comment

---
 src/main/java/org/apache/datasketches/cpc/TestUtil.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/datasketches/cpc/TestUtil.java b/src/main/java/org/apache/datasketches/cpc/TestUtil.java
index 442e3ffe6..e9dad61ab 100644
--- a/src/main/java/org/apache/datasketches/cpc/TestUtil.java
+++ b/src/main/java/org/apache/datasketches/cpc/TestUtil.java
@@ -68,7 +68,7 @@ static boolean specialEquals(final CpcSketch sk1, final CpcSketch sk2,
       rtAssertEquals(sk1.mergeFlag, sk2.mergeFlag);
       rtAssertEquals(ficolA, ficolB);
       rtAssertEquals(sk1.kxp, sk2.kxp, .01 * sk1.kxp); //1% tolerance
-      rtAssertEquals(sk1.hipEstAccum, sk2.hipEstAccum, 01 * sk1.hipEstAccum); //1% tolerance
+      rtAssertEquals(sk1.hipEstAccum, sk2.hipEstAccum, .01 * sk1.hipEstAccum); //1% tolerance
     }
     return true;
   }

From 5a4ddb885c1155bed079f901c4c737c1cbb2221f Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Tue, 8 Apr 2025 17:32:51 -0700
Subject: [PATCH 87/88] Fix 2 typos.

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9022928f8..f321dab4f 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ If you are interested in making contributions to this site please see our [Commu
 
 ### OpenJDK Version 21
 An OpenJDK-compatible build of Java 21, provided by one of the Open-Source providers, such as Azul Systems, Red Hat, SAP, Eclipse Temurin, etc, is required.
-All of the testing of this release has been performed with a Eclipse Temurin build.
+All of the testing of this release has been performed with an Eclipse Temurin build.
 
 This release uses the new Java Foreign Function & Memory (FFM) features that are in "preview" in Java 21.
 As a result, the JVM flag **--enable-preview** must be set at compile and at runtime.
@@ -62,7 +62,7 @@ This DataSketches component is structured as a Maven project and Maven is the re
 #### A Toolchain is required
 
 * You must have a JDK type toolchain defined in location *~/.m2/toolchains.xml* that specifies where to find a locally installed OpenJDK-compatible version 21.
-* Your default \$JAVA\_HOME compiler must be an OpenJDK compatible, specified in the toolchain, and may be a version greater than 21. Note that if your \$JAVA\_HOME is set to a Java version greater than 21, Maven will automatically use the Java 21 version specified in the toolchain instead. The included pom.xml specifies the necessary JVM flags, so no further action should be required.
+* Your default \$JAVA\_HOME compiler must be OpenJDK compatible, specified in the toolchain, and may be a version greater than 21. Note that if your \$JAVA\_HOME is set to a Java version greater than 21, Maven will automatically use the Java 21 version specified in the toolchain instead. The included pom.xml specifies the necessary JVM flags, so no further action should be required.
 * Note that the paths specified in the toolchain must be fully qualified direct paths to the OpenJDK version locations. Using environment variables will not work.
 
 #### To run normal unit tests:

From 45bb98de64920b0927c429387b26d8c73e611c8c Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Wed, 9 Apr 2025 10:27:45 -0700
Subject: [PATCH 88/88] Remove exclusions from FindBugsExcludeFilter.xml

---
 tools/FindBugsExcludeFilter.xml | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/tools/FindBugsExcludeFilter.xml b/tools/FindBugsExcludeFilter.xml
index d47e78f09..eb410b99c 100644
--- a/tools/FindBugsExcludeFilter.xml
+++ b/tools/FindBugsExcludeFilter.xml
@@ -24,17 +24,6 @@ under the License.
   
     
   
-  
-  
-    
-  
-
-  
-     
-    
-    
-    
-  
 
   
@@ -49,7 +38,8 @@ under the License.
     
   
 
-   
+  
+   
     
     
     
@@ -58,7 +48,8 @@ under the License.
     
   
 
-   
+  
+   
     
     
       

  *  * Long || Start Byte Adr:
diff --git a/src/main/java/org/apache/datasketches/quantiles/DoublesByteArrayImpl.java b/src/main/java/org/apache/datasketches/quantiles/DoublesByteArrayImpl.java
index f4df5aa8b..8451bad33 100644
--- a/src/main/java/org/apache/datasketches/quantiles/DoublesByteArrayImpl.java
+++ b/src/main/java/org/apache/datasketches/quantiles/DoublesByteArrayImpl.java
@@ -58,7 +58,7 @@ static byte[] toByteArray(final DoublesSketch sketch, final boolean ordered, fin
         | (ordered ? ORDERED_FLAG_MASK : 0)
         | (compact ? (COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK) : 0);
 
-    if (empty && !sketch.hasMemory()) { //empty & has Memory
+    if (empty && !sketch.hasMemory()) { //empty & !has Memory
       final byte[] outByteArr = new byte[Long.BYTES];
       final WritableMemory memOut = WritableMemory.writableWrap(outByteArr);
       final int preLongs = 1;
@@ -79,15 +79,7 @@ static byte[] toByteArray(final DoublesSketch sketch, final boolean ordered, fin
    */
   private static byte[] convertToByteArray(final DoublesSketch sketch, final int flags,
                                            final boolean ordered, final boolean compact) {
-    final int preLongs = 2;
-    final int extra = 2; // extra space for min and max quantiles
-    final int prePlusExtraBytes = (preLongs + extra) << 3;
-    final int k = sketch.getK();
-    final long n = sketch.getN();
-
-    // If not-compact, have accessor always report full levels. Then use level size to determine
-    // whether to copy data out.
-    final DoublesSketchAccessor dsa = DoublesSketchAccessor.wrap(sketch, !compact);
+    final int preLongs = sketch.isEmpty() ? 1 : 2;
 
     final int outBytes = (compact ? sketch.getCurrentCompactSerializedSizeBytes()
         : sketch.getCurrentUpdatableSerializedSizeBytes());
@@ -95,15 +87,23 @@ private static byte[] convertToByteArray(final DoublesSketch sketch, final int f
     final byte[] outByteArr = new byte[outBytes];
     final WritableMemory memOut = WritableMemory.writableWrap(outByteArr);
 
-    //insert preamble-0, N, min, max
+    //insert pre0
+    final int k = sketch.getK();
     insertPre0(memOut, preLongs, flags, k);
     if (sketch.isEmpty()) { return outByteArr; }
 
+    //insert N, min, max
+    final long n = sketch.getN();
     insertN(memOut, n);
     insertMinDouble(memOut, sketch.isEmpty() ? Double.NaN : sketch.getMinItem());
     insertMaxDouble(memOut, sketch.isEmpty() ? Double.NaN : sketch.getMaxItem());
 
-    long memOffsetBytes = prePlusExtraBytes;
+    // If not-compact, have accessor always report full levels. Then use level size to determine
+    // whether to copy data out.
+    final DoublesSketchAccessor dsa = DoublesSketchAccessor.wrap(sketch, !compact);
+
+    final int minAndMax = 2; // extra space for min and max quantiles
+    long memOffsetBytes = (preLongs + minAndMax) << 3;
 
     // might need to sort base buffer but don't want to change input sketch
     final int bbCnt = computeBaseBufferItems(k, n);
diff --git a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java
index 1e3408511..951bd7244 100644
--- a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java
+++ b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java
@@ -32,6 +32,7 @@
 import org.apache.datasketches.memory.WritableBuffer;
 import org.apache.datasketches.memory.WritableMemory;
 import org.apache.datasketches.quantilescommon.QuantilesAPI;
+import org.apache.datasketches.quantilescommon.QuantilesUtil;
 
 /**
  * t-Digest for estimating quantiles and ranks.
@@ -125,7 +126,7 @@ public void merge(final TDigestDouble other) {
   /**
    * Process buffered values and merge centroids if needed
    */
-  public void compress() {
+  private void compress() {
     if (numBuffered_ == 0) { return; }
     final int num = numBuffered_ + numCentroids_;
     final double[] values =  new double[num];
@@ -277,6 +278,51 @@ public double getQuantile(final double rank) {
     return weightedAverage(centroidWeights_[numCentroids_ - 1], w1, maxValue_, w2);
   }
 
+  /**
+   * Returns an approximation to the Probability Mass Function (PMF) of the input stream
+   * given a set of split points.
+   *
+   * @param splitPoints an array of m unique, monotonically increasing values
+   * that divide the input domain into m+1 consecutive disjoint intervals (bins).
+   *
+   * @return an array of m+1 doubles each of which is an approximation
+   * to the fraction of the input stream values (the mass) that fall into one of those intervals.
+   * @throws SketchesStateException if sketch is empty.
+   */
+  public double[] getPMF(final double[] splitPoints) {
+    final double[] buckets = getCDF(splitPoints);
+    for (int i = buckets.length; i-- > 1; ) {
+      buckets[i] -= buckets[i - 1];
+    }
+    return buckets;
+  }
+
+  /**
+   * Returns an approximation to the Cumulative Distribution Function (CDF), which is the
+   * cumulative analog of the PMF, of the input stream given a set of split points.
+   *
+   * @param splitPoints an array of m unique, monotonically increasing values
+   * that divide the input domain into m+1 consecutive disjoint intervals.
+   *
+   * @return an array of m+1 doubles, which are a consecutive approximation to the CDF
+   * of the input stream given the splitPoints. The value at array position j of the returned
+   * CDF array is the sum of the returned values in positions 0 through j of the returned PMF
+   * array. This can be viewed as array of ranks of the given split points plus one more value
+   * that is always 1.
+   * @throws SketchesStateException if sketch is empty.
+   */
+  public double[] getCDF(final double[] splitPoints) {
+    if (isEmpty()) { throw new SketchesStateException(QuantilesAPI.EMPTY_MSG); }
+    QuantilesUtil.checkDoublesSplitPointsOrder(splitPoints);
+    final int len = splitPoints.length + 1;
+    final double[] ranks = new double[len];
+    for (int i = 0; i < len - 1; i++) {
+      ranks[i] = getRank(splitPoints[i]);
+    }
+    ranks[len - 1] = 1.0;
+    return ranks;
+  }
+
   /**
    * Computes size needed to serialize the current state.
    * @return size in bytes needed to serialize this tdigest
diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java
index 1df6cc9d9..8327a0d5e 100644
--- a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java
+++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java
@@ -139,7 +139,7 @@ public void basicWritableWrapTest() {
   @Test
   public void countWritableWrappedBitsWhenDirty() {
     // like basicOperationTest but with setBit which does
-    // not neecssarily track numBitsSet_
+    // not necessarily track numBitsSet_
     final HeapBitArray hba = new HeapBitArray(128);
     assertFalse(hba.getAndSetBit(1));
     assertFalse(hba.getAndSetBit(2));
diff --git a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java
index 985cdf798..09eebabf7 100644
--- a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java
+++ b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java
@@ -31,7 +31,6 @@
 
 import org.apache.datasketches.memory.DefaultMemoryRequestServer;
 import org.apache.datasketches.memory.Memory;
-//import org.apache.datasketches.memory.WritableHandle;
 import org.apache.datasketches.memory.WritableMemory;
 
 /**
@@ -74,8 +73,6 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool
     byte[] barr1;
     WritableMemory wmem;
     try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes)).scope()) {
-      //byte[] byteArr = new byte[bytes];
-      //WritableMemory wmem = WritableMemory.wrap(byteArr);
       hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
       assertTrue(hllSketch.isEmpty());
 
diff --git a/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java b/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java
index deb8c5be5..17f3d0d0f 100644
--- a/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java
+++ b/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java
@@ -109,7 +109,6 @@ public void checkCorruptMemoryInput() {
     HllSketch sk = new HllSketch(12);
     byte[] memObj = sk.toCompactByteArray();
     WritableMemory wmem = WritableMemory.writableWrap(memObj);
-    //long memAdd = wmem.getCumulativeOffset(0);
     HllSketch bad;
 
     //checkFamily
@@ -148,7 +147,6 @@ public void checkCorruptMemoryInput() {
     for (int i = 1; i <= 15; i++) { sk.update(i); }
     memObj = sk.toCompactByteArray();
     wmem = WritableMemory.writableWrap(memObj);
-    //memAdd = wmem.getCumulativeOffset(0);
 
     //check wrong PreInts and SET
     try {
@@ -162,7 +160,6 @@ public void checkCorruptMemoryInput() {
     for (int i = 15; i <= 1000; i++) { sk.update(i); }
     memObj = sk.toCompactByteArray();
     wmem = WritableMemory.writableWrap(memObj);
-    //memAdd = wmem.getCumulativeOffset(0);
 
     //check wrong PreInts and HLL
     try {
@@ -179,7 +176,6 @@ public void checkExtractFlags() {
     int bytes = HllSketch.getMaxUpdatableSerializationBytes(4, TgtHllType.HLL_4);
     WritableMemory wmem = WritableMemory.allocate(bytes);
     Object memObj = wmem.getArray();
-    //long memAdd = wmem.getCumulativeOffset(0L);
     HllSketch sk = new HllSketch(4, TgtHllType.HLL_4, wmem);
     int flags = extractFlags(wmem);
     assertEquals(flags, EMPTY_FLAG_MASK);
diff --git a/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java b/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java
index 078f3503b..53b422b7c 100644
--- a/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java
@@ -67,6 +67,16 @@ public void generateKllFloatsSketchBinaries() throws IOException {
     }
   }
 
+  @Test(groups = {GENERATE_JAVA_FILES})
+  public void generateKllLongsSketchBinaries() throws IOException {
+    final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000};
+    for (int n: nArr) {
+      final KllLongsSketch sk = KllLongsSketch.newHeapInstance();
+      for (int i = 1; i <= n; i++) { sk.update(i); }
+      Files.newOutputStream(javaPath.resolve("kll_long_n" + n + "_java.sk")).write(sk.toByteArray());
+    }
+  }
+
   @Test(groups = {GENERATE_JAVA_FILES})
   public void generateKllItemsSketchBinaries() throws IOException {
     final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000};
diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectCompactDoublesSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectCompactDoublesSketchTest.java
index 9831c2f57..7a4d061ad 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDirectCompactDoublesSketchTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDirectCompactDoublesSketchTest.java
@@ -110,13 +110,13 @@ public void checkDirectCompactGetDoubleItemsArray() {
 
     KllDoublesSketch sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
     double[] itemsArr = sk2.getDoubleItemsArray();
-    for (int i = 0; i < 20; i++) { assertEquals(itemsArr[i], 0F); }
+    for (int i = 0; i < 20; i++) { assertEquals(itemsArr[i], 0.0); }
 
     sk.update(1);
     sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
     itemsArr = sk2.getDoubleItemsArray();
-    for (int i = 0; i < 19; i++) { assertEquals(itemsArr[i], 0F); }
-    assertEquals(itemsArr[19], 1F);
+    for (int i = 0; i < 19; i++) { assertEquals(itemsArr[i], 0.0); }
+    assertEquals(itemsArr[19], 1.0);
 
     for (int i = 2; i <= 21; i++) { sk.update(i); }
     sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
@@ -169,12 +169,12 @@ public void checkMinAndMax() {
     try { sk2.getMaxItem(); fail(); } catch (SketchesArgumentException e) {}
     sk.update(1);
     sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
-    assertEquals(sk2.getMaxItem(),1.0F);
-    assertEquals(sk2.getMinItem(),1.0F);
+    assertEquals(sk2.getMaxItem(),1.0);
+    assertEquals(sk2.getMinItem(),1.0);
     for (int i = 2; i <= 21; i++) { sk.update(i); }
     sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
-    assertEquals(sk2.getMaxItem(),21.0F);
-    assertEquals(sk2.getMinItem(),1.0F);
+    assertEquals(sk2.getMaxItem(),21.0);
+    assertEquals(sk2.getMinItem(),1.0);
   }
 
   @Test
diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchIteratorTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchIteratorTest.java
index 78a3b9cd5..4bfdfa4fc 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchIteratorTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchIteratorTest.java
@@ -41,7 +41,7 @@ public void oneItemSketch() {
     sketch.update(0);
     QuantilesDoublesSketchIterator it = sketch.iterator();
     Assert.assertTrue(it.next());
-    Assert.assertEquals(it.getQuantile(), 0f);
+    Assert.assertEquals(it.getQuantile(), 0);
     Assert.assertEquals(it.getWeight(), 1);
     Assert.assertFalse(it.next());
   }
diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java
index 33219a806..6342ac33d 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java
@@ -189,11 +189,11 @@ public void mergeLowerK() {
       sketch2.update(2 * n - i - 1);
     }
 
-    assertEquals(sketch1.getMinItem(), 0.0f);
-    assertEquals(sketch1.getMaxItem(), n - 1f);
+    assertEquals(sketch1.getMinItem(), 0.0);
+    assertEquals(sketch1.getMaxItem(), n - 1.0);
 
     assertEquals(sketch2.getMinItem(), n);
-    assertEquals(sketch2.getMaxItem(), 2f * n - 1f);
+    assertEquals(sketch2.getMaxItem(), 2.0 * n - 1.0);
 
     assertTrue(sketch1.getNormalizedRankError(false) < sketch2.getNormalizedRankError(false));
     assertTrue(sketch1.getNormalizedRankError(true) < sketch2.getNormalizedRankError(true));
@@ -613,7 +613,7 @@ public void checkWritableWrapOfCompactForm() {
   public void checkReadOnlyExceptions() {
     int k = 20;
     double[] dblArr = new double[0];
-    double dblV = 1.0f;
+    double dblV = 1.0;
     int idx = 1;
     boolean bool = true;
     KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(k);
diff --git a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchSerDeTest.java b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchSerDeTest.java
index e07a395da..007cc8370 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchSerDeTest.java
@@ -64,7 +64,7 @@ public void serializeDeserializeEmpty() {
   @Test
   public void serializeDeserializeOneValue() {
     final KllDoublesSketch sk1 = KllDoublesSketch.newHeapInstance();
-    sk1.update(1);
+    sk1.update(1.0);
 
     //from heap -> byte[] -> heap
     final byte[] bytes = sk1.toByteArray();
diff --git a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchTest.java
index 0b3818f1f..e143577f4 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchTest.java
@@ -165,8 +165,8 @@ public void manyValuesEstimationMode() {
     assertEquals(pmf[0], 0.5, PMF_EPS_FOR_K_256);
     assertEquals(pmf[1], 0.5, PMF_EPS_FOR_K_256);
 
-    assertEquals(sketch.getMinItem(), 0f); // min value is exact
-    assertEquals(sketch.getMaxItem(), n - 1f); // max value is exact
+    assertEquals(sketch.getMinItem(), 0.0); // min value is exact
+    assertEquals(sketch.getMaxItem(), n - 1.0); // max value is exact
 
     // check at every 0.1 percentage point
     final double[] fractions = new double[1001];
@@ -261,11 +261,11 @@ public void mergeLowerK() {
       sketch2.update(2 * n - i - 1);
     }
 
-    assertEquals(sketch1.getMinItem(), 0.0f);
-    assertEquals(sketch1.getMaxItem(), n - 1f);
+    assertEquals(sketch1.getMinItem(), 0.0);
+    assertEquals(sketch1.getMaxItem(), n - 1);
 
     assertEquals(sketch2.getMinItem(), n);
-    assertEquals(sketch2.getMaxItem(), 2f * n - 1.0);
+    assertEquals(sketch2.getMaxItem(), 2.0 * n - 1.0);
 
     assertTrue(sketch1.getNormalizedRankError(false) < sketch2.getNormalizedRankError(false));
     assertTrue(sketch1.getNormalizedRankError(true) < sketch2.getNormalizedRankError(true));
@@ -306,7 +306,7 @@ public void mergeEmptyLowerK() {
     sketch2.merge(sketch1);
     assertFalse(sketch1.isEmpty());
     assertEquals(sketch1.getN(), n);
-    assertEquals(sketch1.getMinItem(), 0f);
+    assertEquals(sketch1.getMinItem(), 0.0);
     assertEquals(sketch1.getMaxItem(), n - 1.0);
     assertEquals(sketch1.getQuantile(0.5), n / 2.0, n * PMF_EPS_FOR_K_256);
   }
@@ -424,7 +424,7 @@ public void checkNewDirectInstanceAndSize() {
     KllDoublesSketch.newDirectInstance(wmem, memReqSvr);
     try { KllDoublesSketch.newDirectInstance(null, memReqSvr); fail(); }
     catch (NullPointerException e) { }
-    try { KllFloatsSketch.newDirectInstance(wmem, null); fail(); }
+    try { KllDoublesSketch.newDirectInstance(wmem, null); fail(); }
     catch (NullPointerException e) { }
     int updateSize = KllSketch.getMaxSerializedSizeBytes(200, 0, DOUBLES_SKETCH, true);
     int compactSize = KllSketch.getMaxSerializedSizeBytes(200, 0, DOUBLES_SKETCH, false);
diff --git a/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java
index 00028e341..9fc74d97b 100644
--- a/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java
@@ -570,7 +570,7 @@ public void checkCDF_PDF() {
   }
 
   @Test
-  public void checkWrapCase1Floats() {
+  public void checkWrapCase1Items() {
     KllItemsSketch sk = KllItemsSketch.newHeapInstance(20, Comparator.naturalOrder(), serDe);
     final int n = 21;
     final int digits = Util.numDigits(n);
diff --git a/src/test/java/org/apache/datasketches/kll/KllMiscDoublesTest.java b/src/test/java/org/apache/datasketches/kll/KllMiscDoublesTest.java
index e58c27419..4ce988d22 100644
--- a/src/test/java/org/apache/datasketches/kll/KllMiscDoublesTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllMiscDoublesTest.java
@@ -100,8 +100,8 @@ public void checkHeapifyExceptions2() {
   @Test(expectedExceptions = SketchesArgumentException.class)
   public void checkHeapifyExceptions3() {
     KllDoublesSketch sk = KllDoublesSketch.newHeapInstance();
-    sk.update(1.0f);
-    sk.update(2.0f);
+    sk.update(1.0);
+    sk.update(2.0);
     WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray());
     wmem.putByte(0, (byte) 1); //corrupt preamble ints, should be 5
     KllDoublesSketch.heapify(wmem);
diff --git a/src/test/java/org/apache/datasketches/kll/KllMiscItemsTest.java b/src/test/java/org/apache/datasketches/kll/KllMiscItemsTest.java
index acf3343d9..5f51a7f1a 100644
--- a/src/test/java/org/apache/datasketches/kll/KllMiscItemsTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllMiscItemsTest.java
@@ -293,7 +293,7 @@ public void checkSketchInitializeItemsHeap() {
     final int digits = Util.numDigits(n);
     KllItemsSketch sk;
 
-    println("#### CASE: FLOAT FULL HEAP");
+    println("#### CASE: ITEM FULL HEAP");
     sk = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
     for (int i = 1; i <= n; i++) { sk.update(Util.longToFixedLengthString(i, digits)); }
     println(sk.toString(true, true));
@@ -310,7 +310,7 @@ public void checkSketchInitializeItemsHeap() {
     assertEquals(sk.getNumLevels(), 2);
     assertFalse(sk.isLevelZeroSorted());
 
-    println("#### CASE: FLOAT HEAP EMPTY");
+    println("#### CASE: ITEM HEAP EMPTY");
     sk = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
     println(sk.toString(true, true));
     assertEquals(sk.getK(), k);
@@ -326,7 +326,7 @@ public void checkSketchInitializeItemsHeap() {
     assertEquals(sk.getNumLevels(), 1);
     assertFalse(sk.isLevelZeroSorted());
 
-    println("#### CASE: FLOAT HEAP SINGLE");
+    println("#### CASE: ITEM HEAP SINGLE");
     sk = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
     sk.update("1");
     println(sk.toString(true, true));
@@ -354,7 +354,7 @@ public void checkSketchInitializeItemsHeapifyCompactMem() {
     byte[] compBytes;
     Memory mem;
 
-    println("#### CASE: FLOAT FULL HEAPIFIED FROM COMPACT");
+    println("#### CASE: ITEM FULL HEAPIFIED FROM COMPACT");
     sk2 = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
     for (int i = 1; i <= n; i++) { sk2.update(Util.longToFixedLengthString(i, digits)); }
     println(sk2.toString(true, true));
@@ -375,7 +375,7 @@ public void checkSketchInitializeItemsHeapifyCompactMem() {
     assertEquals(sk.getNumLevels(), 2);
     assertFalse(sk.isLevelZeroSorted());
 
-    println("#### CASE: FLOAT EMPTY HEAPIFIED FROM COMPACT");
+    println("#### CASE: ITEM EMPTY HEAPIFIED FROM COMPACT");
     sk2 =  KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
     //println(sk.toString(true, true));
     compBytes = sk2.toByteArray();
@@ -395,7 +395,7 @@ public void checkSketchInitializeItemsHeapifyCompactMem() {
     assertEquals(sk.getNumLevels(), 1);
     assertFalse(sk.isLevelZeroSorted());
 
-    println("#### CASE: FLOAT SINGLE HEAPIFIED FROM COMPACT");
+    println("#### CASE: ITEM SINGLE HEAPIFIED FROM COMPACT");
     sk2 = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
     sk2.update("1");
     //println(sk2.toString(true, true));
@@ -417,7 +417,7 @@ public void checkSketchInitializeItemsHeapifyCompactMem() {
     assertFalse(sk.isLevelZeroSorted());
   }
 
-  //public void checkSketchInitializeFloatHeapifyUpdatableMem() Not Supported
+  //public void checkSketchInitializeItemHeapifyUpdatableMem() Not Supported
 
   @Test //set static enablePrinting = true for visual checking
   public void checkMemoryToStringItemsCompact() {
@@ -431,7 +431,7 @@ public void checkMemoryToStringItemsCompact() {
     Memory mem;
     String s;
 
-    println("#### CASE: FLOAT FULL COMPACT");
+    println("#### CASE: ITEM FULL COMPACT");
     sk = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
     for (int i = 1; i <= n; i++) { sk.update(Util.longToFixedLengthString(i, digits)); }
     compBytes = sk.toByteArray();
@@ -447,7 +447,7 @@ public void checkMemoryToStringItemsCompact() {
     println(s);
     assertEquals(compBytes, compBytes2);
 
-    println("#### CASE: FLOAT EMPTY COMPACT");
+    println("#### CASE: ITEM EMPTY COMPACT");
     sk = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
     compBytes = sk.toByteArray();
     mem = Memory.wrap(compBytes);
@@ -462,7 +462,7 @@ public void checkMemoryToStringItemsCompact() {
     println(s);
     assertEquals(compBytes, compBytes2);
 
-    println("#### CASE: FLOAT SINGLE COMPACT");
+    println("#### CASE: ITEM SINGLE COMPACT");
     sk = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
     sk.update("1");
     compBytes = sk.toByteArray();
diff --git a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java
index b1bd5818d..636105ef8 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java
@@ -31,7 +31,6 @@
 import jdk.incubator.foreign.ResourceScope;
 
 import org.apache.datasketches.memory.DefaultMemoryRequestServer;
-//import org.apache.datasketches.memory.WritableHandle;
 import org.apache.datasketches.memory.WritableMemory;
 import org.apache.datasketches.quantilescommon.QuantilesDoublesSketchIterator;
 
diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
index c252eef3c..d896bbefa 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
@@ -30,7 +30,6 @@
 import org.testng.annotations.Test;
 
 import org.apache.datasketches.memory.Memory;
-//import org.apache.datasketches.memory.WritableHandle;
 import org.apache.datasketches.memory.WritableMemory;
 
 import jdk.incubator.foreign.ResourceScope;
diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
index ea928ba02..8cdc7bf71 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
@@ -141,7 +141,6 @@ public void checkEmptyExceptions() {
 
   @Test
   public void directSketchShouldMoveOntoHeapEventually() {
-
     WritableMemory wmem = WritableMemory.allocateDirect(1000, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
     WritableMemory wmem2 = wmem;
     UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
diff --git a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java
index db043cff6..e1c6914c7 100644
--- a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java
+++ b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java
@@ -41,6 +41,8 @@ public void empty() {
     assertThrows(SketchesStateException.class, () -> td.getMaxValue());
     assertThrows(SketchesStateException.class, () -> td.getRank(0));
     assertThrows(SketchesStateException.class, () -> td.getQuantile(0.5));
+    assertThrows(SketchesStateException.class, () -> td.getPMF(new double[]{0}));
+    assertThrows(SketchesStateException.class, () -> td.getCDF(new double[]{0}));
   }
 
   @Test
@@ -82,6 +84,14 @@ public void manyValues() {
     assertEquals(td.getQuantile(0.9), n * 0.9, n * 0.9 * 0.01);
     assertEquals(td.getQuantile(0.95), n * 0.95, n * 0.95 * 0.01);
     assertEquals(td.getQuantile(1), n - 1);
+    final double[] pmf = td.getPMF(new double[] {n / 2});
+    assertEquals(pmf.length, 2);
+    assertEquals(pmf[0], 0.5, 0.0001);
+    assertEquals(pmf[1], 0.5, 0.0001);
+    final double[] cdf = td.getCDF(new double[] {n / 2});
+    assertEquals(cdf.length, 2);
+    assertEquals(cdf[0], 0.5, 0.0001);
+    assertEquals(cdf[1], 1.0);
   }
 
   @Test
diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
index 1e2089b51..188dbf427 100644
--- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
@@ -32,7 +32,6 @@
 import org.apache.datasketches.common.SketchesArgumentException;
 import org.apache.datasketches.memory.DefaultMemoryRequestServer;
 import org.apache.datasketches.memory.Memory;
-//import org.apache.datasketches.memory.WritableHandle;
 import org.apache.datasketches.memory.WritableMemory;
 import org.testng.annotations.Test;
 
diff --git a/src/test/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketchTest.java
index e4c112281..6d6af7047 100644
--- a/src/test/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketchTest.java
@@ -119,7 +119,6 @@ public void checkHeapifyByteArrayExact() {
     // That is, this is being run for its side-effect of accessing things.
     // If something is wonky, it will generate an exception and fail the test.
     local2.toString(true, true, 8, true);
-
   }
 
   @Test
diff --git a/src/test/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketchTest.java
index 2261edc3b..84ddcb80e 100644
--- a/src/test/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketchTest.java
@@ -114,7 +114,7 @@ public void checkIllegalSketchID_UpdateSketch() {
     WritableMemory mem = WritableMemory.writableWrap(byteArray);
     mem.putByte(FAMILY_BYTE, (byte) 0); //corrupt the Sketch ID byte
 
-    //try to heapify the corruped mem
+    //try to heapify the corrupted mem
     Sketch.heapify(mem, sl.seed);
   }
 
diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
index ef6004a1b..f36597b7c 100644
--- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
@@ -47,7 +47,6 @@
 import org.apache.datasketches.common.SketchesReadOnlyException;
 import org.apache.datasketches.memory.DefaultMemoryRequestServer;
 import org.apache.datasketches.memory.Memory;
-//import org.apache.datasketches.memory.WritableHandle;
 import org.apache.datasketches.memory.WritableMemory;
 import org.apache.datasketches.thetacommon.HashOperations;
 import org.apache.datasketches.thetacommon.ThetaUtil;
@@ -86,7 +85,7 @@ public void checkBadSerVer() {
     }
   }
 
-  @Test//(expectedExceptions = SketchesArgumentException.class)
+  @Test
   public void checkConstructorKtooSmall() {
     int k = 8;
     WritableMemory wmem;
@@ -98,7 +97,7 @@ public void checkConstructorKtooSmall() {
     }
   }
 
-  @Test//(expectedExceptions = SketchesArgumentException.class)
+  @Test
   public void checkConstructorMemTooSmall() {
     int k = 16;
     WritableMemory wmem;
@@ -195,7 +194,7 @@ public void checkWrapIllegalFamilyID_direct() {
     DirectQuickSelectSketch.writableWrap(mem, ThetaUtil.DEFAULT_UPDATE_SEED);
   }
 
-  @Test //(expectedExceptions = SketchesArgumentException.class)
+  @Test
   public void checkHeapifySeedConflict() {
     int k = 512;
     long seed1 = 1021;
@@ -213,7 +212,7 @@ public void checkHeapifySeedConflict() {
     }
   }
 
-  @Test//(expectedExceptions = SketchesArgumentException.class)
+  @Test
   public void checkCorruptLgNomLongs() {
     int k = 16;
     WritableMemory wmem;
diff --git a/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java b/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java
index 5e81b1808..c0cff6eb2 100644
--- a/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java
+++ b/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java
@@ -41,29 +41,29 @@ public void checkHeapifyCompactSketchAssumedDefaultSeed() {
     final int k = 64;
     final long seed = ThetaUtil.DEFAULT_UPDATE_SEED;
     final short seedHash = Util.computeSeedHash(seed);
-    UpdateSketch sv3usk = UpdateSketch.builder().setNominalEntries(k).setSeed(seed).build();
-    for (int i=0; i
     
     
-    
+    
 
     
       

From 14174086dfdac3b8b586b9cf2d3aaaf7b7cabacb Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Wed, 18 Dec 2024 14:51:14 -0800
Subject: [PATCH 32/88] update GHA workflows

---
 .github/workflows/auto-check_cpp_files.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/auto-check_cpp_files.yml b/.github/workflows/auto-check_cpp_files.yml
index b9e05ae97..4b06687e0 100644
--- a/.github/workflows/auto-check_cpp_files.yml
+++ b/.github/workflows/auto-check_cpp_files.yml
@@ -12,33 +12,33 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      
+
 	  - name: Checkout C++
         uses: actions/checkout@v4
         with:
           repository: apache/datasketches-cpp
           path: cpp
-      
+
 	  - name: Setup Java
         uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'temurin'
-      
+
 	  - name: Configure C++ build
         run: cd cpp/build && cmake .. -DGENERATE=true
-      
+
 	  - name: Build C++ unit tests
         run: cd cpp && cmake --build build --config Release
-      
+
 	  - name: Run C++ tests
         run: cd cpp && cmake --build build --config Release --target test
-      
+
 	  - name: Make dir
         run: mkdir -p serialization_test_data/cpp_generated_files
-      
+
       - name: Copy files
         run: cp cpp/build/*/test/*_cpp.sk serialization_test_data/cpp_generated_files
-      
+
 	  - name: Run Java tests
         run: mvn test -P check-cpp-files

From 341de43f1ee347621020785091d8a7c81344d68f Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Wed, 18 Dec 2024 15:26:32 -0800
Subject: [PATCH 33/88] update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 547df3e9d..b6db7f89e 100644
--- a/README.md
+++ b/README.md
@@ -53,7 +53,7 @@ and, as a result, must be compiled with JDK17 and this dependency:
 
 If your application only relies on the APIs of datasketches-java no special JVM arguments are required.
 However, if your application also directly relies on the APIs of the *datasketches-memory* component, 
-you may need the additional JVM argument **--enable-preview**.
+you may need the additional JVM argument **--add-modules=jdk.incubator.foreign**.
 
 ### Recommended Build Tool
 This DataSketches component is structured as a Maven project and Maven is the recommended Build Tool.

From babbe129bd1a994709cbbfe702596117b88d4919 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Wed, 18 Dec 2024 15:54:59 -0800
Subject: [PATCH 34/88] Release Process: update pom to 7.0.0

update the version for the maven-javadoc-plugin and the
versions-maven-plugin
---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 4c5532e54..0ab6e9fcc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,7 +33,7 @@ under the License.
 
   org.apache.datasketches
   datasketches-java
-  7.0.0-SNAPSHOT
+  7.0.0
   jar
 
   ${project.artifactId}
@@ -128,7 +128,7 @@ under the License.
     
     0.8.12
     
-    2.17.1
+    2.18.0
     
     1.0.0
   

From 5819b40fb53edaaf47538a0787bcf02437fe4e5d Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Wed, 18 Dec 2024 15:58:30 -0800
Subject: [PATCH 35/88] fix maven-javadoc-plugin version

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 0ab6e9fcc..35022e3b1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,7 +112,7 @@ under the License.
     3.5.0
     3.2.7
     3.4.2
-    3.11.1
+    3.11.2
     3.1.1
     3.2.0
     3.3.1

From dc232771abcd402b335f906958c335a14c606e78 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Wed, 18 Dec 2024 17:49:20 -0800
Subject: [PATCH 36/88] Correct indentation in GHAW auto-check_cpp_files.yml

---
 .github/workflows/auto-check_cpp_files.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/auto-check_cpp_files.yml b/.github/workflows/auto-check_cpp_files.yml
index 4b06687e0..068b12f4d 100644
--- a/.github/workflows/auto-check_cpp_files.yml
+++ b/.github/workflows/auto-check_cpp_files.yml
@@ -13,32 +13,32 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-	  - name: Checkout C++
+      - name: Checkout C++
         uses: actions/checkout@v4
         with:
           repository: apache/datasketches-cpp
           path: cpp
 
-	  - name: Setup Java
+      - name: Setup Java
         uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'temurin'
 
-	  - name: Configure C++ build
+      - name: Configure C++ build
         run: cd cpp/build && cmake .. -DGENERATE=true
 
-	  - name: Build C++ unit tests
+      - name: Build C++ unit tests
         run: cd cpp && cmake --build build --config Release
 
-	  - name: Run C++ tests
+      - name: Run C++ tests
         run: cd cpp && cmake --build build --config Release --target test
 
-	  - name: Make dir
+      - name: Make dir
         run: mkdir -p serialization_test_data/cpp_generated_files
 
       - name: Copy files
         run: cp cpp/build/*/test/*_cpp.sk serialization_test_data/cpp_generated_files
 
-	  - name: Run Java tests
+      - name: Run Java tests
         run: mvn test -P check-cpp-files

From 868d33d696bcfef755bc9a605905421e246640f1 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Thu, 19 Dec 2024 15:17:49 -0800
Subject: [PATCH 37/88] update pom version on main

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 35022e3b1..c72af4f39 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,7 +33,7 @@ under the License.
 
   org.apache.datasketches
   datasketches-java
-  7.0.0
+  7.1.0-SNAPSHOT
   jar
 
   ${project.artifactId}

From 763c58a5fd0ed006a52ae1bdd439efdca8e5dfbd Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Sat, 28 Dec 2024 12:53:14 -0800
Subject: [PATCH 38/88] Fix workflows in main.

---
 .github/workflows/auto-jdk-matrix.yml         | 100 ++++++++-------
 .github/workflows/auto-os-matrix.yml          | 118 +++++++++---------
 ...heck_cpp_files.yml => check_cpp_files.yml} |   7 +-
 .github/workflows/codeql-analysis.yml         |  83 ++++++++++++
 .../{manual-javadoc.yml => javadoc.yml}       |   0
 .github/workflows/manual-codeql-analysis.yml  |  57 ---------
 6 files changed, 202 insertions(+), 163 deletions(-)
 rename .github/workflows/{auto-check_cpp_files.yml => check_cpp_files.yml} (70%)
 create mode 100644 .github/workflows/codeql-analysis.yml
 rename .github/workflows/{manual-javadoc.yml => javadoc.yml} (100%)
 delete mode 100644 .github/workflows/manual-codeql-analysis.yml

diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml
index f8220bbe2..9312589df 100644
--- a/.github/workflows/auto-jdk-matrix.yml
+++ b/.github/workflows/auto-jdk-matrix.yml
@@ -1,67 +1,71 @@
 name: DataSketches-Java Auto JDK Matrix Test & Install
 
 on:
-    pull_request:
-    push:
-        branches: [ master, main ]
-    workflow_dispatch:
+  push:
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
+  pull_request:
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    # The branches below must be a subset of the branches above
+    branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
+  workflow_dispatch:
 
 env:
-    MAVEN_OPTS: -Xmx4g -Xms1g
+  MAVEN_OPTS: -Xmx4g -Xms1g
 
 jobs:
-    build:
-        name: Build, Test, Install
-        runs-on: ubuntu-latest
+  build:
+    name: Build, Test, Install
+    runs-on: ubuntu-latest
 
-        strategy:
-          fail-fast: false
-          matrix:
-              jdk: [ 17 ]
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [ 17 ]
 
-        env:
-          JDK_VERSION: ${{ matrix.jdk }}
+      env:
+        JDK_VERSION: ${{ matrix.jdk }}
 
-        steps:
-        - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-          uses: actions/checkout@v4
-          with:
-              persist-credentials: false
+      steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
-        - name: Print Current workflow
-          run: >
-             cat .github/workflows/auto-jdk-matrix.yml
+      - name: Print Current workflow
+        run: >
+          cat .github/workflows/auto-jdk-matrix.yml
 
-        - name: Cache local Maven repository
-          uses: actions/cache@v4
-          with:
-              path: ~/.m2/repository
-              key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-              restore-keys: build-${{ runner.os }}-maven-
+      - name: Cache local Maven repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: build-${{ runner.os }}-maven-
 
-        - name: Install Matrix JDK
-          uses: actions/setup-java@v4
-          with:
-              java-version: ${{ matrix.jdk }}
-              distribution: 'temurin'
-              java-package: jdk
-              architecture: x64 
+      - name: Install Matrix JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.jdk }}
+          distribution: 'temurin'
+          java-package: jdk
+          architecture: x64 
 
-        - name: Echo Java Version
-          run: >
-              java -version
+      - name: Echo Java Version
+        run: >
+          java -version
 
-        - name: Test
-          run: >
-              mvn clean test -B
-              -Dmaven.javadoc.skip=true
-              -Dgpg.skip=true
+      - name: Test
+        run: >
+          mvn clean test -B
+          -Dmaven.javadoc.skip=true
+          -Dgpg.skip=true
 
-        - name: Install
-          run: >
-              mvn clean install -B
-              -DskipTests=true
-              -Dgpg.skip=true
+      - name: Install
+        run: >
+          mvn clean install -B
+          -DskipTests=true
+          -Dgpg.skip=true
 
 # Architecture options: x86, x64, armv7, aarch64, ppc64le
 # setup-java@v4 has a "with cache" option
diff --git a/.github/workflows/auto-os-matrix.yml b/.github/workflows/auto-os-matrix.yml
index d9f5bc4c5..9fb6f2812 100644
--- a/.github/workflows/auto-os-matrix.yml
+++ b/.github/workflows/auto-os-matrix.yml
@@ -1,77 +1,81 @@
 name: DataSketches-Java Auto OS Matrix Test & Install
 
 on:
-    pull_request:
-    push:
-        branches: [ master, main ]
-    workflow_dispatch:
+  push:
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
+  pull_request:
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    # The branches below must be a subset of the branches above
+    branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
+  workflow_dispatch:
 
 env:
-    MAVEN_OPTS: -Xmx1g -Xms1g
+  MAVEN_OPTS: -Xmx4g -Xms1g
 
 jobs:
-    build:
-        name: Build, Test, Install
+  build:
+    name: Build, Test, Install
 
-        strategy:
-          fail-fast: false
+    strategy:
+      fail-fast: false
 
-          matrix:
-            jdk: [ 17 ]
-            os: [ windows-latest, ubuntu-latest, macos-latest ]
-            include:
-              - os: windows-latest
-                skip_javadoc: "`-Dmaven`.javadoc`.skip=true"
-                skip_gpg: "`-Dgpg`.skip=true"
-              - os: ubuntu-latest
-                skip_javadoc: -Dmaven.javadoc.skip=true
-                skip_gpg: -Dgpg.skip=true
-              - os: macos-latest
-                skip_javadoc: -Dmaven.javadoc.skip=true
-                skip_gpg: -Dgpg.skip=true
+      matrix:
+        jdk: [ 17 ]
+        os: [ windows-latest, ubuntu-latest, macos-latest ]
+        include:
+          - os: windows-latest
+            skip_javadoc: "`-Dmaven`.javadoc`.skip=true"
+            skip_gpg: "`-Dgpg`.skip=true"
+          - os: ubuntu-latest
+            skip_javadoc: -Dmaven.javadoc.skip=true
+            skip_gpg: -Dgpg.skip=true
+          - os: macos-latest
+            skip_javadoc: -Dmaven.javadoc.skip=true
+            skip_gpg: -Dgpg.skip=true
 
-        runs-on: ${{matrix.os}}
+    runs-on: ${{matrix.os}}
 
-        env:
-          JDK_VERSION: ${{ matrix.jdk }}
+    env:
+      JDK_VERSION: ${{ matrix.jdk }}
 
-        steps:
-        - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-          uses: actions/checkout@v4
-          with:
-              persist-credentials: false
+    steps:
+    - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+      uses: actions/checkout@v4
+      with:
+        persist-credentials: false
 
-        - name: Cache local Maven repository
-          uses: actions/cache@v4
-          with:
-              path: ~/.m2/repository
-              key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-              restore-keys: build-${{ runner.os }}-maven-
+    - name: Cache local Maven repository
+      uses: actions/cache@v4
+      with:
+        path: ~/.m2/repository
+        key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: build-${{ runner.os }}-maven-
 
-        - name: Install Matrix JDK
-          uses: actions/setup-java@v4
-          with:
-              java-version: ${{ matrix.jdk }}
-              distribution: 'temurin'
-              java-package: jdk
-              architecture: x64 
+    - name: Install Matrix JDK
+      uses: actions/setup-java@v4
+      with:
+        java-version: ${{ matrix.jdk }}
+        distribution: 'temurin'
+        java-package: jdk
+        architecture: x64 
 
-        - name: Echo Java Version
-          run: >
-              java -version
+    - name: Echo Java Version
+      run: >
+        java -version
 
-        - name: Test
-          run: >
-              mvn clean test
-              ${{matrix.os.skip_javadoc}}
-              ${{matrix.os.skip_gpg}}
+    - name: Test
+      run: >
+        mvn clean test
+        ${{matrix.os.skip_javadoc}}
+        ${{matrix.os.skip_gpg}}
 
-        - name: Install
-          run: >
-              mvn clean install -B
-              ${{matrix.os.skip_javadoc}}
-              -D skipTests=true
-              ${{matrix.os.skip_gpg}}
+    - name: Install
+      run: >
+        mvn clean install -B
+        ${{matrix.os.skip_javadoc}}
+        -D skipTests=true
+        ${{matrix.os.skip_gpg}}
 
 # Architecture options: x86, x64, armv7, aarch64, ppc64le
 # setup-java@v4 has a "with cache" option
diff --git a/.github/workflows/auto-check_cpp_files.yml b/.github/workflows/check_cpp_files.yml
similarity index 70%
rename from .github/workflows/auto-check_cpp_files.yml
rename to .github/workflows/check_cpp_files.yml
index 068b12f4d..4ca9e0aa4 100644
--- a/.github/workflows/auto-check_cpp_files.yml
+++ b/.github/workflows/check_cpp_files.yml
@@ -2,7 +2,12 @@ name: Serialization Compatibility Test
 
 on:
   push:
-    branches: [ master, main ]
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
+  pull_request:
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    # The branches below must be a subset of the branches above
+    branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
   workflow_dispatch:
 
 jobs:
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
new file mode 100644
index 000000000..e6589b243
--- /dev/null
+++ b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,83 @@
+name: "CodeQL"
+
+on:
+  push:
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
+  pull_request:
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    # The branches below must be a subset of the branches above
+    branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
+  workflow_dispatch:
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'java' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Setup Java
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        cache: 'maven'
+        java-version: '17'
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        queries: +security-and-quality
+        
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # Details on CodeQL's query packs refer to link below. 
+
+    - name: Custom building using maven
+      run: >
+        mvn clean package -f "pom.xml" -B -V -e
+        -Dfindbugs.skip=true
+        -Dcheckstyle.skip=true
+        -Dpmd.skip=true
+        -Denforcer.skip
+        -Dmaven.javadoc.skip
+        -DskipTests=true
+        -Dmaven.test.skip.exec
+        -Dlicense.skip=true
+        -Dweb.console.skip=true
+        -Dgpg.skip=true
+        
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"
+
+
+# CodeQL's Query Packs:
+# https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+
+# Command-line programs to run using the OS shell.
+# See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+# Architecture options: x86, x64, armv7, aarch64, ppc64le
+# Lifecycles: validate, compile, test, package, verify, install, deploy
+# -B batch mode, never stops for user input
+# -V show Version without stopping
+# -X debug mode
+# -q quiet, only show errors
+# -e produce execution error messages
\ No newline at end of file
diff --git a/.github/workflows/manual-javadoc.yml b/.github/workflows/javadoc.yml
similarity index 100%
rename from .github/workflows/manual-javadoc.yml
rename to .github/workflows/javadoc.yml
diff --git a/.github/workflows/manual-codeql-analysis.yml b/.github/workflows/manual-codeql-analysis.yml
deleted file mode 100644
index 1b94d13a3..000000000
--- a/.github/workflows/manual-codeql-analysis.yml
+++ /dev/null
@@ -1,57 +0,0 @@
-name: "CodeQL"
-
-on:
-  workflow_dispatch:
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'java' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v4
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        queries: +security-and-quality
-
-        
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v4
-
-    #   Command-line programs to run using the OS shell.
-    #   See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v4
-      with:
-        category: "/language:${{matrix.language}}"

From 4ace77e9fa449825be917610c248bf8edb919d1a Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Thu, 2 Jan 2025 10:45:01 -0800
Subject: [PATCH 39/88] change step indentations

---
 .github/workflows/auto-jdk-matrix.yml | 66 +++++++++++++--------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml
index 9312589df..7468a3e0e 100644
--- a/.github/workflows/auto-jdk-matrix.yml
+++ b/.github/workflows/auto-jdk-matrix.yml
@@ -27,45 +27,45 @@ jobs:
         JDK_VERSION: ${{ matrix.jdk }}
 
       steps:
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
+        - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+          uses: actions/checkout@v4
+          with:
+            persist-credentials: false
 
-      - name: Print Current workflow
-        run: >
-          cat .github/workflows/auto-jdk-matrix.yml
+        - name: Print Current workflow
+          run: >
+            cat .github/workflows/auto-jdk-matrix.yml
 
-      - name: Cache local Maven repository
-        uses: actions/cache@v4
-        with:
-          path: ~/.m2/repository
-          key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: build-${{ runner.os }}-maven-
+        - name: Cache local Maven repository
+          uses: actions/cache@v4
+          with:
+            path: ~/.m2/repository
+            key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+            restore-keys: build-${{ runner.os }}-maven-
 
-      - name: Install Matrix JDK
-        uses: actions/setup-java@v4
-        with:
-          java-version: ${{ matrix.jdk }}
-          distribution: 'temurin'
-          java-package: jdk
-          architecture: x64 
+        - name: Install Matrix JDK
+          uses: actions/setup-java@v4
+          with:
+            java-version: ${{ matrix.jdk }}
+            distribution: 'temurin'
+            java-package: jdk
+            architecture: x64 
 
-      - name: Echo Java Version
-        run: >
-          java -version
+        - name: Echo Java Version
+          run: >
+            java -version
 
-      - name: Test
-        run: >
-          mvn clean test -B
-          -Dmaven.javadoc.skip=true
-          -Dgpg.skip=true
+        - name: Test
+          run: >
+            mvn clean test -B
+            -Dmaven.javadoc.skip=true
+            -Dgpg.skip=true
 
-      - name: Install
-        run: >
-          mvn clean install -B
-          -DskipTests=true
-          -Dgpg.skip=true
+        - name: Install
+          run: >
+            mvn clean install -B
+            -DskipTests=true
+            -Dgpg.skip=true
 
 # Architecture options: x86, x64, armv7, aarch64, ppc64le
 # setup-java@v4 has a "with cache" option

From f5b99618f24cc048450ef0dc4b20ab99ebf20889 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Thu, 2 Jan 2025 13:35:34 -0800
Subject: [PATCH 40/88] repair indentations

---
 .github/workflows/auto-jdk-matrix.yml | 68 +++++++++++++--------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml
index 7468a3e0e..5adbbbe08 100644
--- a/.github/workflows/auto-jdk-matrix.yml
+++ b/.github/workflows/auto-jdk-matrix.yml
@@ -26,46 +26,46 @@ jobs:
       env:
         JDK_VERSION: ${{ matrix.jdk }}
 
-      steps:
-        - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-          uses: actions/checkout@v4
-          with:
-            persist-credentials: false
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
-        - name: Print Current workflow
-          run: >
-            cat .github/workflows/auto-jdk-matrix.yml
+      - name: Print Current workflow
+        run: >
+          cat .github/workflows/auto-jdk-matrix.yml
 
-        - name: Cache local Maven repository
-          uses: actions/cache@v4
-          with:
-            path: ~/.m2/repository
-            key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-            restore-keys: build-${{ runner.os }}-maven-
+      - name: Cache local Maven repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: build-${{ runner.os }}-maven-
 
-        - name: Install Matrix JDK
-          uses: actions/setup-java@v4
-          with:
-            java-version: ${{ matrix.jdk }}
-            distribution: 'temurin'
-            java-package: jdk
-            architecture: x64 
+      - name: Install Matrix JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.jdk }}
+          distribution: 'temurin'
+          java-package: jdk
+          architecture: x64 
 
-        - name: Echo Java Version
-          run: >
-            java -version
+      - name: Echo Java Version
+        run: >
+          java -version
 
-        - name: Test
-          run: >
-            mvn clean test -B
-            -Dmaven.javadoc.skip=true
-            -Dgpg.skip=true
+      - name: Test
+        run: >
+          mvn clean test -B
+          -Dmaven.javadoc.skip=true
+          -Dgpg.skip=true
 
-        - name: Install
-          run: >
-            mvn clean install -B
-            -DskipTests=true
-            -Dgpg.skip=true
+      - name: Install
+        run: >
+          mvn clean install -B
+          -DskipTests=true
+          -Dgpg.skip=true
 
 # Architecture options: x86, x64, armv7, aarch64, ppc64le
 # setup-java@v4 has a "with cache" option

From 1fd781f5eb7500e5db5465e15282016da9eddb15 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Thu, 2 Jan 2025 16:38:42 -0800
Subject: [PATCH 41/88] fix indents again

---
 .github/workflows/auto-jdk-matrix.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml
index 5adbbbe08..49c7cf1dc 100644
--- a/.github/workflows/auto-jdk-matrix.yml
+++ b/.github/workflows/auto-jdk-matrix.yml
@@ -23,8 +23,8 @@ jobs:
       matrix:
         jdk: [ 17 ]
 
-      env:
-        JDK_VERSION: ${{ matrix.jdk }}
+    env:
+      JDK_VERSION: ${{ matrix.jdk }}
 
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"

From 35526df5935776f14b4a2fd18019f8e42e4d6683 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Thu, 2 Jan 2025 16:52:43 -0800
Subject: [PATCH 42/88] adjust indents & titles

---
 .github/workflows/auto-jdk-matrix.yml |  4 +-
 .github/workflows/auto-os-matrix.yml  | 64 +++++++++++++--------------
 2 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml
index 49c7cf1dc..9043932ce 100644
--- a/.github/workflows/auto-jdk-matrix.yml
+++ b/.github/workflows/auto-jdk-matrix.yml
@@ -1,4 +1,4 @@
-name: DataSketches-Java Auto JDK Matrix Test & Install
+name: Auto JDK Matrix Test & Install
 
 on:
   push:
@@ -24,7 +24,7 @@ jobs:
         jdk: [ 17 ]
 
     env:
-      JDK_VERSION: ${{ matrix.jdk }}
+      addJDK_VERSION: ${{ matrix.jdk }}
 
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
diff --git a/.github/workflows/auto-os-matrix.yml b/.github/workflows/auto-os-matrix.yml
index 9fb6f2812..b56e5aa22 100644
--- a/.github/workflows/auto-os-matrix.yml
+++ b/.github/workflows/auto-os-matrix.yml
@@ -1,4 +1,4 @@
-name: DataSketches-Java Auto OS Matrix Test & Install
+name: Auto OS Matrix Test & Install
 
 on:
   push:
@@ -40,42 +40,42 @@ jobs:
       JDK_VERSION: ${{ matrix.jdk }}
 
     steps:
-    - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-      uses: actions/checkout@v4
-      with:
-        persist-credentials: false
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
-    - name: Cache local Maven repository
-      uses: actions/cache@v4
-      with:
-        path: ~/.m2/repository
-        key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-        restore-keys: build-${{ runner.os }}-maven-
+      - name: Cache local Maven repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: build-${{ runner.os }}-maven-
 
-    - name: Install Matrix JDK
-      uses: actions/setup-java@v4
-      with:
-        java-version: ${{ matrix.jdk }}
-        distribution: 'temurin'
-        java-package: jdk
-        architecture: x64 
+      - name: Install Matrix JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.jdk }}
+          distribution: 'temurin'
+          java-package: jdk
+          architecture: x64 
 
-    - name: Echo Java Version
-      run: >
-        java -version
+      - name: Echo Java Version
+        run: >
+          java -version
 
-    - name: Test
-      run: >
-        mvn clean test
-        ${{matrix.os.skip_javadoc}}
-        ${{matrix.os.skip_gpg}}
+      - name: Test
+        run: >
+          mvn clean test
+          ${{matrix.os.skip_javadoc}}
+          ${{matrix.os.skip_gpg}}
 
-    - name: Install
-      run: >
-        mvn clean install -B
-        ${{matrix.os.skip_javadoc}}
-        -D skipTests=true
-        ${{matrix.os.skip_gpg}}
+      - name: Install
+        run: >
+          mvn clean install -B
+          ${{matrix.os.skip_javadoc}}
+          -D skipTests=true
+          ${{matrix.os.skip_gpg}}
 
 # Architecture options: x86, x64, armv7, aarch64, ppc64le
 # setup-java@v4 has a "with cache" option

From e412cfd5e67414d905e246f8b2cd83d72994f47b Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Thu, 2 Jan 2025 17:04:04 -0800
Subject: [PATCH 43/88] correct indents

---
 .github/workflows/check_cpp_files.yml |  2 +-
 .github/workflows/codeql-analysis.yml | 60 +++++++++++++--------------
 2 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/check_cpp_files.yml b/.github/workflows/check_cpp_files.yml
index 4ca9e0aa4..00a9e5255 100644
--- a/.github/workflows/check_cpp_files.yml
+++ b/.github/workflows/check_cpp_files.yml
@@ -1,4 +1,4 @@
-name: Serialization Compatibility Test
+name: CPP SerDe Compatibility Test
 
 on:
   push:
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index e6589b243..28c17a1f6 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -27,45 +27,45 @@ jobs:
         # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-    - name: Setup Java
-      uses: actions/setup-java@v4
-      with:
-        distribution: 'temurin'
-        cache: 'maven'
-        java-version: '17'
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          cache: 'maven'
+          java-version: '17'
 
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
-      with:
-        languages: ${{ matrix.language }}
-        queries: +security-and-quality
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+         languages: ${{ matrix.language }}
+         queries: +security-and-quality
         
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
         # Details on CodeQL's query packs refer to link below. 
 
-    - name: Custom building using maven
-      run: >
-        mvn clean package -f "pom.xml" -B -V -e
-        -Dfindbugs.skip=true
-        -Dcheckstyle.skip=true
-        -Dpmd.skip=true
-        -Denforcer.skip
-        -Dmaven.javadoc.skip
-        -DskipTests=true
-        -Dmaven.test.skip.exec
-        -Dlicense.skip=true
-        -Dweb.console.skip=true
-        -Dgpg.skip=true
+      - name: Custom building using maven
+        run: >
+          mvn clean package -f "pom.xml" -B -V -e
+          -Dfindbugs.skip=true
+          -Dcheckstyle.skip=true
+          -Dpmd.skip=true
+          -Denforcer.skip
+          -Dmaven.javadoc.skip
+          -DskipTests=true
+          -Dmaven.test.skip.exec
+          -Dlicense.skip=true
+          -Dweb.console.skip=true
+          -Dgpg.skip=true
         
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
-      with:
-        category: "/language:${{matrix.language}}"
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+         category: "/language:${{matrix.language}}"
 
 
 # CodeQL's Query Packs:

From 62939a21b8d085a6fa9beff5eaa9f8080aac8e4d Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Thu, 2 Jan 2025 17:24:35 -0800
Subject: [PATCH 44/88] fix cat of workflow

---
 .github/workflows/javadoc.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/javadoc.yml b/.github/workflows/javadoc.yml
index 5f6ac6aec..fc1507a2c 100644
--- a/.github/workflows/javadoc.yml
+++ b/.github/workflows/javadoc.yml
@@ -22,7 +22,7 @@ jobs:
 
       - name: Print Current workflow
         run: >
-         cat .github/workflows/manual-javadoc.yml
+         cat .github/workflows/javadoc.yml
 
       - name: Generate JavaDoc
         run: mvn javadoc:javadoc

From fd0ede4f063358ed573f24155ce15adf6e7f6847 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Thu, 2 Jan 2025 17:30:42 -0800
Subject: [PATCH 45/88] fix target directory path

---
 .github/workflows/javadoc.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/javadoc.yml b/.github/workflows/javadoc.yml
index fc1507a2c..0381f43b8 100644
--- a/.github/workflows/javadoc.yml
+++ b/.github/workflows/javadoc.yml
@@ -31,6 +31,6 @@ jobs:
         uses: JamesIves/github-pages-deploy-action@v4.6.8
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          folder: target/site/apidocs
+          folder: target/reports/apidocs
           target-folder: docs/${{ github.ref_name }}
           branch: gh-pages

From b5d2d04fd36ab7c71fc45e532a2ddc4b29dbe72d Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Thu, 2 Jan 2025 18:05:44 -0800
Subject: [PATCH 46/88] fix typo

---
 .github/workflows/auto-jdk-matrix.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml
index 9043932ce..7e1d26c70 100644
--- a/.github/workflows/auto-jdk-matrix.yml
+++ b/.github/workflows/auto-jdk-matrix.yml
@@ -24,7 +24,7 @@ jobs:
         jdk: [ 17 ]
 
     env:
-      addJDK_VERSION: ${{ matrix.jdk }}
+      JDK_VERSION: ${{ matrix.jdk }}
 
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"

From 16a00a97c6309b6574889d1b255cbbb8a3bca399 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Fri, 3 Jan 2025 16:53:46 -0800
Subject: [PATCH 47/88] Remove two obsolete files.

Update Sketches Checkstyle.xml
---
 tools/CloverConfig.txt       | 50 ------------------------------------
 tools/SketchesCheckstyle.xml | 15 +++++------
 tools/suppressions.xml       | 29 ---------------------
 3 files changed, 6 insertions(+), 88 deletions(-)
 delete mode 100644 tools/CloverConfig.txt
 delete mode 100644 tools/suppressions.xml

diff --git a/tools/CloverConfig.txt b/tools/CloverConfig.txt
deleted file mode 100644
index 845fede8c..000000000
--- a/tools/CloverConfig.txt
+++ /dev/null
@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-Clover Config for Eclipse:
-
-At Project Level Properties:
-Clover:
-	Instrumentation:
-		Initstring: Default value
-		Output Folder: ...project output dir(s)
-		Flush Policy: At JVM shutdown ...
-		Misc: Fully qualify ... , Instrument and compile at statement level
-	Contexts:
-		Check: assert statements
-		Add Custom Coverage Context Filter:
-			private-constructor: also see link below
-				Method
-				(.* )?private +[a-zA-Z0-9_$]+ *\( *\).*
-	Source Files
-		Only look ...
-			[check] src/main/java[includes=**/*.java][excludes=]
-			[check] src/test/java[includes=**/*.java][excludes=]
-	Test Classes
-		Assume all source in the specified folders are tests or test utility classes
-			[check] src/test/java
-
-At Clover "down-triangle" menu:
-	Columns:
-		Element
-		% TOTAL Coverage
-		Uncovered Elements: Custom: %UncoveredElements * TotalElements / 100
-		Total Elements
-
-
-
-http://alexruizlog.blogspot.com/2009/04/how-to-make-clover-ignore-private_21.html
\ No newline at end of file
diff --git a/tools/SketchesCheckstyle.xml b/tools/SketchesCheckstyle.xml
index 184a05ad3..53726a2c1 100644
--- a/tools/SketchesCheckstyle.xml
+++ b/tools/SketchesCheckstyle.xml
@@ -49,10 +49,12 @@ under the License.
     
   
 
+  
   
-    
+    
   
   
+  
   
   
   
@@ -67,6 +69,7 @@ under the License.
     
   
   
+  
   
     
     
@@ -80,13 +83,6 @@ under the License.
     
     
     
-  
-    
 
     
     
@@ -230,7 +226,8 @@ under the License.
       
     
     
-    
+    
+    
 
     
       
diff --git a/tools/suppressions.xml b/tools/suppressions.xml
deleted file mode 100644
index 91b8833c4..000000000
--- a/tools/suppressions.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-  
-
-
-  
-  
-
\ No newline at end of file

From 54e48e55128282a8a7bf8657a55cb03c413de73e Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Fri, 3 Jan 2025 16:55:46 -0800
Subject: [PATCH 48/88] Add LF at end of file.

---
 .github/workflows/codeql-analysis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 28c17a1f6..734fbb60f 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -80,4 +80,4 @@ jobs:
 # -V show Version without stopping
 # -X debug mode
 # -q quiet, only show errors
-# -e produce execution error messages
\ No newline at end of file
+# -e produce execution error messages

From 2d2b69d170ca4b09b1d74b407cecfabda5f2a198 Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Thu, 9 Jan 2025 11:20:15 -0800
Subject: [PATCH 49/88] fixed bit packing

---
 .../apache/datasketches/theta/BitPacking.java |  4 +-
 .../datasketches/theta/BitPackingTest.java    | 92 ++++++++++---------
 2 files changed, 49 insertions(+), 47 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/theta/BitPacking.java b/src/main/java/org/apache/datasketches/theta/BitPacking.java
index ca70dafa2..de914f908 100644
--- a/src/main/java/org/apache/datasketches/theta/BitPacking.java
+++ b/src/main/java/org/apache/datasketches/theta/BitPacking.java
@@ -482,7 +482,7 @@ static void packBits13(final long[] values, final int i, final byte[] buf, int o
 
     buf[off++] = (byte) (values[i + 3] >>> 4);
 
-    buf[off] = (byte) (values[i + 3] >>> 4);
+    buf[off] = (byte) (values[i + 3] << 4);
     buf[off++] |= values[i + 4] >>> 9;
 
     buf[off++] = (byte) (values[i + 4] >>> 1);
@@ -4449,7 +4449,7 @@ static void unpackBits35(final long[] values, final int i, final byte[] buf, int
     values[i + 1] |= Byte.toUnsignedLong(buf[off++]) << 6;
     values[i + 1] |= Byte.toUnsignedLong(buf[off]) >>> 2;
 
-    values[i + 2] = (Byte.toUnsignedLong(buf[off++]) & 2) << 33;
+    values[i + 2] = (Byte.toUnsignedLong(buf[off++]) & 3) << 33;
     values[i + 2] |= (Byte.toUnsignedLong(buf[off++])) << 25;
     values[i + 2] |= Byte.toUnsignedLong(buf[off++]) << 17;
     values[i + 2] |= Byte.toUnsignedLong(buf[off++]) << 9;
diff --git a/src/test/java/org/apache/datasketches/theta/BitPackingTest.java b/src/test/java/org/apache/datasketches/theta/BitPackingTest.java
index 152bee039..3ed6062e7 100644
--- a/src/test/java/org/apache/datasketches/theta/BitPackingTest.java
+++ b/src/test/java/org/apache/datasketches/theta/BitPackingTest.java
@@ -31,60 +31,62 @@ public class BitPackingTest {
 
   @Test
   public void packUnpackBits() {
-    for (int bits = 1; bits <= 63; bits++) {
-      final long mask = (1 << bits) - 1;
-      long[] input = new long[8];
-      final long golden64 = Util.INVERSE_GOLDEN_U64;
-      long value = 0xaa55aa55aa55aa55L; // arbitrary starting value
-      for (int i = 0; i < 8; ++i) {
-        input[i] = value & mask;
-        value += golden64;
-      }
-      byte[] bytes = new byte[8 * Long.BYTES];
-      int bitOffset = 0;
-      int bufOffset = 0;
-      for (int i = 0; i < 8; ++i) {
-        BitPacking.packBits(input[i], bits, bytes, bufOffset, bitOffset);
-        bufOffset += (bitOffset + bits) >>> 3;
-        bitOffset = (bitOffset + bits) & 7;
-      }
+    long value = 0xaa55aa55aa55aa55L; // arbitrary starting value
+    for (int n = 0; n < 100; n++) {
+      for (int bits = 1; bits <= 63; bits++) {
+        final long mask = (1 << bits) - 1;
+        long[] input = new long[8];
+        for (int i = 0; i < 8; ++i) {
+          input[i] = value & mask;
+          value += Util.INVERSE_GOLDEN_U64;
+        }
+        byte[] bytes = new byte[8 * Long.BYTES];
+        int bitOffset = 0;
+        int bufOffset = 0;
+        for (int i = 0; i < 8; ++i) {
+          BitPacking.packBits(input[i], bits, bytes, bufOffset, bitOffset);
+          bufOffset += (bitOffset + bits) >>> 3;
+          bitOffset = (bitOffset + bits) & 7;
+        }
 
-      long[] output = new long[8];
-      bitOffset = 0;
-      bufOffset = 0;
-      for (int i = 0; i < 8; ++i) {
-        BitPacking.unpackBits(output, i, bits, bytes, bufOffset, bitOffset);
-        bufOffset += (bitOffset + bits) >>> 3;
-        bitOffset = (bitOffset + bits) & 7;
-      }
-      for (int i = 0; i < 8; ++i) {
-        assertEquals(output[i], input[i]);
+        long[] output = new long[8];
+        bitOffset = 0;
+        bufOffset = 0;
+        for (int i = 0; i < 8; ++i) {
+          BitPacking.unpackBits(output, i, bits, bytes, bufOffset, bitOffset);
+          bufOffset += (bitOffset + bits) >>> 3;
+          bitOffset = (bitOffset + bits) & 7;
+        }
+        for (int i = 0; i < 8; ++i) {
+          assertEquals(output[i], input[i]);
+        }
       }
     }
   }
 
   @Test
   public void packUnpackBlocks() {
-    for (int bits = 1; bits <= 63; bits++) {
-      if (enablePrinting) { System.out.println("bits " + bits); }
-      final long mask = (1L << bits) - 1;
-      long[] input = new long[8];
-      final long golden64 = Util.INVERSE_GOLDEN_U64;
-      long value = 0xaa55aa55aa55aa55L; // arbitrary starting value
-      for (int i = 0; i < 8; ++i) {
-        input[i] = value & mask;
-        value += golden64;
-      }
-      byte[] bytes = new byte[8 * Long.BYTES];
-      BitPacking.packBitsBlock8(input, 0, bytes, 0, bits);
-      if (enablePrinting) { hexDump(bytes); }
+    long value = 0xaa55aa55aa55aa55L; // arbitrary starting value
+    for (int n = 0; n < 100; n++) {
+      for (int bits = 1; bits <= 63; bits++) {
+        if (enablePrinting) { System.out.println("bits " + bits); }
+        final long mask = (1L << bits) - 1;
+        long[] input = new long[8];
+        for (int i = 0; i < 8; ++i) {
+          input[i] = value & mask;
+          value += Util.INVERSE_GOLDEN_U64;
+        }
+        byte[] bytes = new byte[8 * Long.BYTES];
+        BitPacking.packBitsBlock8(input, 0, bytes, 0, bits);
+        if (enablePrinting) { hexDump(bytes); }
 
-      long[] output = new long[8];
-      BitPacking.unpackBitsBlock8(output, 0, bytes, 0, bits);
+        long[] output = new long[8];
+        BitPacking.unpackBitsBlock8(output, 0, bytes, 0, bits);
 
-      for (int i = 0; i < 8; ++i) {
-        if (enablePrinting) { System.out.println("checking value " + i); }
-        assertEquals(output[i], input[i]);
+        for (int i = 0; i < 8; ++i) {
+          if (enablePrinting) { System.out.println("checking value " + i); }
+          assertEquals(output[i], input[i]);
+        }
       }
     }
   }

From b1122dd005aeed872edc04dcdf9584a659e0dfa4 Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Fri, 10 Jan 2025 10:47:12 -0800
Subject: [PATCH 50/88] better error messages

---
 src/main/java/org/apache/datasketches/theta/BitPacking.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/theta/BitPacking.java b/src/main/java/org/apache/datasketches/theta/BitPacking.java
index de914f908..99bcfb105 100644
--- a/src/main/java/org/apache/datasketches/theta/BitPacking.java
+++ b/src/main/java/org/apache/datasketches/theta/BitPacking.java
@@ -152,7 +152,7 @@ static void packBitsBlock8(final long[] values, final int i, final byte[] buf, f
       case 61: packBits61(values, i, buf, off); break;
       case 62: packBits62(values, i, buf, off); break;
       case 63: packBits63(values, i, buf, off); break;
-      default: throw new SketchesArgumentException("wrong number of bits " + bits);
+      default: throw new SketchesArgumentException("wrong number of bits in packBitsBlock8: " + bits);
     }
   }
 
@@ -221,7 +221,7 @@ static void unpackBitsBlock8(final long[] values, final int i, final byte[] buf,
       case 61: unpackBits61(values, i, buf, off); break;
       case 62: unpackBits62(values, i, buf, off); break;
       case 63: unpackBits63(values, i, buf, off); break;
-      default: throw new SketchesArgumentException("wrong number of bits " + bits);
+      default: throw new SketchesArgumentException("wrong number of bits unpackBitsBlock8: " + bits);
     }
   }
 

From 2d2579ed20a17d98482b07c728dc2738f3c38de1 Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Fri, 10 Jan 2025 10:47:58 -0800
Subject: [PATCH 51/88] more rounds of pseudo-random data

---
 .../java/org/apache/datasketches/theta/BitPackingTest.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/datasketches/theta/BitPackingTest.java b/src/test/java/org/apache/datasketches/theta/BitPackingTest.java
index 3ed6062e7..a961bffc0 100644
--- a/src/test/java/org/apache/datasketches/theta/BitPackingTest.java
+++ b/src/test/java/org/apache/datasketches/theta/BitPackingTest.java
@@ -32,7 +32,7 @@ public class BitPackingTest {
   @Test
   public void packUnpackBits() {
     long value = 0xaa55aa55aa55aa55L; // arbitrary starting value
-    for (int n = 0; n < 100; n++) {
+    for (int n = 0; n < 10000; n++) {
       for (int bits = 1; bits <= 63; bits++) {
         final long mask = (1 << bits) - 1;
         long[] input = new long[8];
@@ -67,7 +67,7 @@ public void packUnpackBits() {
   @Test
   public void packUnpackBlocks() {
     long value = 0xaa55aa55aa55aa55L; // arbitrary starting value
-    for (int n = 0; n < 100; n++) {
+    for (int n = 0; n < 10000; n++) {
       for (int bits = 1; bits <= 63; bits++) {
         if (enablePrinting) { System.out.println("bits " + bits); }
         final long mask = (1L << bits) - 1;

From 545427946b8744450ccca0f0218d88d1042c86de Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Mon, 13 Jan 2025 18:56:49 -0800
Subject: [PATCH 52/88] run javadoc action on push to main

---
 .github/workflows/javadoc.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/javadoc.yml b/.github/workflows/javadoc.yml
index 0381f43b8..ff724afaa 100644
--- a/.github/workflows/javadoc.yml
+++ b/.github/workflows/javadoc.yml
@@ -1,6 +1,8 @@
 name: JavaDoc
 
 on:
+  puah:
+    branches: main
   workflow_dispatch:
 
 jobs:

From 5bc58655705ec793a6f54c31178034ee90122703 Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Mon, 13 Jan 2025 19:14:43 -0800
Subject: [PATCH 53/88] typo fix

---
 .github/workflows/javadoc.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/javadoc.yml b/.github/workflows/javadoc.yml
index ff724afaa..bd5903836 100644
--- a/.github/workflows/javadoc.yml
+++ b/.github/workflows/javadoc.yml
@@ -1,7 +1,7 @@
 name: JavaDoc
 
 on:
-  puah:
+  push:
     branches: main
   workflow_dispatch:
 

From 3b20aebe013c93536cf3ada5f21d4689a4789bac Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Tue, 14 Jan 2025 14:11:25 -0800
Subject: [PATCH 54/88] removed pattern that seems to be problematic

---
 .github/workflows/check_cpp_files.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/check_cpp_files.yml b/.github/workflows/check_cpp_files.yml
index 00a9e5255..feac7a55a 100644
--- a/.github/workflows/check_cpp_files.yml
+++ b/.github/workflows/check_cpp_files.yml
@@ -2,10 +2,10 @@ name: CPP SerDe Compatibility Test
 
 on:
   push:
-    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ]
     branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
   pull_request:
-    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ]
     # The branches below must be a subset of the branches above
     branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
   workflow_dispatch:

From 731a251cf43bb4d88bed78036c1efb4492f705af Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Tue, 14 Jan 2025 16:36:50 -0800
Subject: [PATCH 55/88] fix workflows ignore_paths

---
 .github/workflows/auto-jdk-matrix.yml | 4 ++--
 .github/workflows/auto-os-matrix.yml  | 4 ++--
 .github/workflows/check_cpp_files.yml | 4 ++--
 .github/workflows/codeql-analysis.yml | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml
index 7e1d26c70..8c20cc742 100644
--- a/.github/workflows/auto-jdk-matrix.yml
+++ b/.github/workflows/auto-jdk-matrix.yml
@@ -2,10 +2,10 @@ name: Auto JDK Matrix Test & Install
 
 on:
   push:
-    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ]
     branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
   pull_request:
-    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ]
     # The branches below must be a subset of the branches above
     branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
   workflow_dispatch:
diff --git a/.github/workflows/auto-os-matrix.yml b/.github/workflows/auto-os-matrix.yml
index b56e5aa22..65be62f98 100644
--- a/.github/workflows/auto-os-matrix.yml
+++ b/.github/workflows/auto-os-matrix.yml
@@ -2,10 +2,10 @@ name: Auto OS Matrix Test & Install
 
 on:
   push:
-    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ]
     branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
   pull_request:
-    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ]
     # The branches below must be a subset of the branches above
     branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
   workflow_dispatch:
diff --git a/.github/workflows/check_cpp_files.yml b/.github/workflows/check_cpp_files.yml
index 00a9e5255..feac7a55a 100644
--- a/.github/workflows/check_cpp_files.yml
+++ b/.github/workflows/check_cpp_files.yml
@@ -2,10 +2,10 @@ name: CPP SerDe Compatibility Test
 
 on:
   push:
-    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ]
     branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
   pull_request:
-    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ]
     # The branches below must be a subset of the branches above
     branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
   workflow_dispatch:
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 734fbb60f..1102d3149 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -2,10 +2,10 @@ name: "CodeQL"
 
 on:
   push:
-    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ]
     branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
   pull_request:
-    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/.*', '**/LICENSE', '**/NOTICE' ]
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ]
     # The branches below must be a subset of the branches above
     branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
   workflow_dispatch:

From 982afa409c16c8218950a27732bf2ec5c4ed45df Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Tue, 14 Jan 2025 17:58:21 -0800
Subject: [PATCH 56/88] update notice

---
 NOTICE | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/NOTICE b/NOTICE
index 8381c8de1..2ac24f2d3 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,9 +1,9 @@
 Apache DataSketches Java
-Copyright 2024 The Apache Software Foundation
+Copyright 2025 The Apache Software Foundation
 
 Copyright 2015-2018 Yahoo Inc.
 Copyright 2019-2020 Verizon Media
-Copyright 2021 Yahoo Inc.
+Copyright 2021-     Yahoo Inc.
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

From b70332aaf325e32f90f417f8c7579fccb58ec99a Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Fri, 17 Jan 2025 13:52:46 -0800
Subject: [PATCH 57/88] removed unused imports

---
 .../datasketches/filters/bloomfilter/BloomFilterTest.java      | 3 ---
 .../java/org/apache/datasketches/hll/DirectAuxHashMapTest.java | 2 --
 .../java/org/apache/datasketches/hll/DirectCouponListTest.java | 1 -
 .../java/org/apache/datasketches/quantiles/DebugUnionTest.java | 2 --
 .../quantiles/DirectQuantilesMemoryRequestTest.java            | 2 --
 .../java/org/apache/datasketches/theta/CompactSketchTest.java  | 3 ---
 .../apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java  | 3 ---
 7 files changed, 16 deletions(-)

diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java
index 25bef3643..7d72fae2f 100644
--- a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java
+++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java
@@ -24,12 +24,9 @@
 import static org.testng.Assert.assertThrows;
 import static org.testng.Assert.assertTrue;
 
-import java.nio.ByteOrder;
-
 import org.apache.datasketches.common.Family;
 import org.apache.datasketches.common.SketchesArgumentException;
 import org.apache.datasketches.common.SketchesReadOnlyException;
-import org.apache.datasketches.memory.DefaultMemoryRequestServer;
 import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.memory.WritableMemory;
 import org.testng.annotations.Test;
diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
index c913af378..75504bc83 100644
--- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
+++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
@@ -35,8 +35,6 @@
 import org.apache.datasketches.memory.WritableMemory;
 import org.testng.annotations.Test;
 
-import jdk.incubator.foreign.ResourceScope;
-
 /**
  * @author Lee Rhodes
  */
diff --git a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java
index 09eebabf7..1b00e0301 100644
--- a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java
+++ b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java
@@ -29,7 +29,6 @@
 
 import jdk.incubator.foreign.ResourceScope;
 
-import org.apache.datasketches.memory.DefaultMemoryRequestServer;
 import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.memory.WritableMemory;
 
diff --git a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java
index 636105ef8..e51be4b20 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java
@@ -23,14 +23,12 @@
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 
-import java.nio.ByteOrder;
 import java.util.HashSet;
 
 import org.testng.annotations.Test;
 
 import jdk.incubator.foreign.ResourceScope;
 
-import org.apache.datasketches.memory.DefaultMemoryRequestServer;
 import org.apache.datasketches.memory.WritableMemory;
 import org.apache.datasketches.quantilescommon.QuantilesDoublesSketchIterator;
 
diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
index d896bbefa..5195f4187 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
@@ -32,8 +32,6 @@
 import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.memory.WritableMemory;
 
-import jdk.incubator.foreign.ResourceScope;
-
 /**
  * The concept for these tests is that the "MemoryManager" classes below are proxies for the
  * implementation that owns the native memory allocations, thus is responsible for
diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
index 188dbf427..e0f79087b 100644
--- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
@@ -26,11 +26,8 @@
 import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 
-import java.nio.ByteOrder;
-
 import org.apache.datasketches.common.Family;
 import org.apache.datasketches.common.SketchesArgumentException;
-import org.apache.datasketches.memory.DefaultMemoryRequestServer;
 import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.memory.WritableMemory;
 import org.testng.annotations.Test;
diff --git a/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java b/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java
index c0cff6eb2..3d4e20fe7 100644
--- a/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java
+++ b/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java
@@ -23,15 +23,12 @@
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 
-import org.apache.datasketches.memory.DefaultMemoryRequestServer;
 import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.memory.WritableMemory;
 import org.apache.datasketches.thetacommon.ThetaUtil;
 import org.apache.datasketches.tuple.Util;
 import org.testng.annotations.Test;
 
-import jdk.incubator.foreign.ResourceScope;
-
 @SuppressWarnings("resource")
 public class HeapifyWrapSerVer1and2Test {
   private static final short defaultSeedHash = Util.computeSeedHash(ThetaUtil.DEFAULT_UPDATE_SEED);

From adff89079a8439ce18ab373a34ab1730a8aa8f07 Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Sun, 26 Jan 2025 13:26:35 -0800
Subject: [PATCH 58/88] use iteration in set ops, wrap compressed sketch and
 unpack in iterator

---
 .../apache/datasketches/theta/AnotBimpl.java  | 34 ++++++++---
 .../datasketches/theta/CompactOperations.java |  2 +-
 .../datasketches/theta/CompactSketch.java     | 18 +++---
 .../theta/DirectCompactSketch.java            | 12 +---
 .../datasketches/theta/IntersectionImpl.java  | 59 +++++++++++--------
 .../datasketches/theta/PreambleUtil.java      |  3 +
 .../org/apache/datasketches/theta/Sketch.java | 13 ++--
 .../apache/datasketches/theta/UnionImpl.java  | 55 ++++-------------
 8 files changed, 92 insertions(+), 104 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/theta/AnotBimpl.java b/src/main/java/org/apache/datasketches/theta/AnotBimpl.java
index e7b2c99eb..6d4c54fd8 100644
--- a/src/main/java/org/apache/datasketches/theta/AnotBimpl.java
+++ b/src/main/java/org/apache/datasketches/theta/AnotBimpl.java
@@ -20,8 +20,11 @@
 package org.apache.datasketches.theta;
 
 import static org.apache.datasketches.common.Util.exactLog2OfLong;
-import static org.apache.datasketches.thetacommon.HashOperations.convertToHashTable;
+import static org.apache.datasketches.thetacommon.HashOperations.checkThetaCorruption;
+import static org.apache.datasketches.thetacommon.HashOperations.continueCondition;
 import static org.apache.datasketches.thetacommon.HashOperations.hashSearch;
+import static org.apache.datasketches.thetacommon.HashOperations.hashSearchOrInsert;
+import static org.apache.datasketches.thetacommon.HashOperations.minLgHashTableSize;
 
 import java.util.Arrays;
 
@@ -124,7 +127,7 @@ public CompactSketch aNotB(final Sketch skA, final Sketch skB, final boolean dst
 
     if (skB.isEmpty()) {
       return skA.compact(dstOrdered, dstMem);
-   }
+    }
     ThetaUtil.checkSeedHashes(skB.getSeedHash(), seedHash_);
     //Both skA & skB are not empty
 
@@ -162,14 +165,12 @@ private static long[] getResultHashArr( //returns a new array
       final long[] hashArrA,
       final Sketch skB) {
 
-    //Rebuild/get hashtable of skB
+    // Rebuild or get hashtable of skB
     final long[] hashTableB; //read only
-    final long[] thetaCache = skB.getCache();
-    final int countB = skB.getRetainedEntries(true);
     if (skB instanceof CompactSketch) {
-      hashTableB = convertToHashTable(thetaCache, countB, minThetaLong, ThetaUtil.REBUILD_THRESHOLD);
+      hashTableB = convertToHashTable(skB, minThetaLong, ThetaUtil.REBUILD_THRESHOLD);
     } else {
-      hashTableB = thetaCache;
+      hashTableB = skB.getCache();
     }
 
     //build temporary result arrays of skA
@@ -191,6 +192,25 @@ private static long[] getResultHashArr( //returns a new array
     return Arrays.copyOfRange(tmpHashArrA, 0, nonMatches);
   }
 
+  private static long[] convertToHashTable(
+      final Sketch sketch,
+      final long thetaLong,
+      final double rebuildThreshold) {
+    final int lgArrLongs = minLgHashTableSize(sketch.getRetainedEntries(true), rebuildThreshold);
+    final int arrLongs = 1 << lgArrLongs;
+    final long[] hashTable = new long[arrLongs];
+    checkThetaCorruption(thetaLong);
+    HashIterator it = sketch.iterator();
+    while (it.next()) {
+      final long hash = it.get();
+      if (continueCondition(thetaLong, hash) ) {
+        continue;
+      }
+      hashSearchOrInsert(hashTable, lgArrLongs, hash);
+    }
+    return hashTable;
+  }
+
   private void reset() {
     thetaLong_ = Long.MAX_VALUE;
     empty_ = true;
diff --git a/src/main/java/org/apache/datasketches/theta/CompactOperations.java b/src/main/java/org/apache/datasketches/theta/CompactOperations.java
index a8066314d..2b52f59fa 100644
--- a/src/main/java/org/apache/datasketches/theta/CompactOperations.java
+++ b/src/main/java/org/apache/datasketches/theta/CompactOperations.java
@@ -161,7 +161,7 @@ static CompactSketch memoryToCompact(
       final long hash = srcMem.getLong(srcPreLongs << 3);
       final SingleItemSketch sis = new SingleItemSketch(hash, srcSeedHash);
       if (dstMem != null) {
-        dstMem.putByteArray(0, sis.toByteArray(),0, 16);
+        dstMem.putByteArray(0, sis.toByteArray(), 0, 16);
         return new DirectCompactSketch(dstMem);
       } else { //heap
         return sis;
diff --git a/src/main/java/org/apache/datasketches/theta/CompactSketch.java b/src/main/java/org/apache/datasketches/theta/CompactSketch.java
index 1426368f1..688ad2746 100644
--- a/src/main/java/org/apache/datasketches/theta/CompactSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/CompactSketch.java
@@ -32,6 +32,7 @@
 import static org.apache.datasketches.theta.PreambleUtil.extractEntryBitsV4;
 import static org.apache.datasketches.theta.PreambleUtil.extractNumEntriesBytesV4;
 import static org.apache.datasketches.theta.PreambleUtil.extractThetaLongV4;
+import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits;
 import static org.apache.datasketches.theta.SingleItemSketch.otherCheckForSingleItem;
 
 import org.apache.datasketches.common.Family;
@@ -189,7 +190,8 @@ private static CompactSketch wrap(final Memory srcMem, final long seed, final bo
     if (serVer == 4) {
       // not wrapping the compressed format since currently we cannot take advantage of
       // decompression during iteration because set operations reach into memory directly
-      return heapifyV4(srcMem, seed, enforceSeed);
+      return DirectCompactCompressedSketch.wrapInstance(srcMem,
+          enforceSeed ? seedHash : (short) extractSeedHash(srcMem));
     }
     else if (serVer == 3) {
       if (PreambleUtil.isEmptyFlag(srcMem)) {
@@ -274,10 +276,6 @@ private int computeMinLeadingZeros() {
     return Long.numberOfLeadingZeros(ored);
   }
 
-  private static int wholeBytesToHoldBits(final int bits) {
-    return (bits >>> 3) + ((bits & 7) > 0 ? 1 : 0);
-  }
-
   private byte[] toByteArrayV4() {
     final int preambleLongs = isEstimationMode() ? 2 : 1;
     final int entryBits = 64 - computeMinLeadingZeros();
@@ -286,8 +284,8 @@ private byte[] toByteArrayV4() {
     // store num_entries as whole bytes since whole-byte blocks will follow (most probably)
     final int numEntriesBytes = wholeBytesToHoldBits(32 - Integer.numberOfLeadingZeros(getRetainedEntries()));
 
-    final int size = preambleLongs * Long.BYTES + numEntriesBytes + wholeBytesToHoldBits(compressedBits);
-    final byte[] bytes = new byte[size];
+    final int sizeBytes = preambleLongs * Long.BYTES + numEntriesBytes + wholeBytesToHoldBits(compressedBits);
+    final byte[] bytes = new byte[sizeBytes];
     final WritableMemory mem = WritableMemory.writableWrap(bytes);
     int offsetBytes = 0;
     mem.putByte(offsetBytes++, (byte) preambleLongs);
@@ -334,12 +332,10 @@ private byte[] toByteArrayV4() {
 
   private static CompactSketch heapifyV4(final Memory srcMem, final long seed, final boolean enforceSeed) {
     final int preLongs = extractPreLongs(srcMem);
-    final int flags = extractFlags(srcMem);
     final int entryBits = extractEntryBitsV4(srcMem);
     final int numEntriesBytes = extractNumEntriesBytesV4(srcMem);
     final short seedHash = (short) extractSeedHash(srcMem);
-    final boolean isEmpty = (flags & EMPTY_FLAG_MASK) > 0;
-    if (enforceSeed && !isEmpty) { PreambleUtil.checkMemorySeedHash(srcMem, seed); }
+    if (enforceSeed) { PreambleUtil.checkMemorySeedHash(srcMem, seed); }
     int offsetBytes = 8;
     long theta = Long.MAX_VALUE;
     if (preLongs > 1) {
@@ -374,7 +370,7 @@ private static CompactSketch heapifyV4(final Memory srcMem, final long seed, fin
       entries[i] += previous;
       previous = entries[i];
     }
-    return new HeapCompactSketch(entries, isEmpty, seedHash, numEntries, theta, true);
+    return new HeapCompactSketch(entries, false, seedHash, numEntries, theta, true);
   }
 
 }
diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java
index 0f69ec3c2..1714d2161 100644
--- a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java
@@ -86,11 +86,7 @@ public int getCurrentBytes() {
 
   @Override
   public double getEstimate() {
-    if (otherCheckForSingleItem(mem_)) { return 1; }
-    final int preLongs = extractPreLongs(mem_);
-    final int curCount = (preLongs == 1) ? 0 : extractCurCount(mem_);
-    final long thetaLong = (preLongs > 2) ? extractThetaLong(mem_) : Long.MAX_VALUE;
-    return Sketch.estimate(thetaLong, curCount);
+    return Sketch.estimate(getThetaLong(), getRetainedEntries());
   }
 
   @Override
@@ -142,10 +138,8 @@ public HashIterator iterator() {
 
   @Override
   public byte[] toByteArray() {
-    final int curCount = getRetainedEntries(true);
-    checkIllegalCurCountAndEmpty(isEmpty(), curCount);
-    final int preLongs = extractPreLongs(mem_);
-    final int outBytes = (curCount + preLongs) << 3;
+    checkIllegalCurCountAndEmpty(isEmpty(), getRetainedEntries());
+    final int outBytes = getCurrentBytes();
     final byte[] byteArrOut = new byte[outBytes];
     mem_.getByteArray(0, byteArrOut, 0, outBytes);
     return byteArrOut;
diff --git a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java
index fc81d1124..3404a0b11 100644
--- a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java
+++ b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java
@@ -288,7 +288,7 @@ else if (curCount_ < 0 && sketchInEntries > 0) {
       else { //On the heap, allocate a HT
         hashTable_ = new long[1 << lgArrLongs_];
       }
-      moveDataToTgt(sketchIn.getCache(), curCount_);
+      moveDataToTgt(sketchIn);
     } //end of state 5
 
     //state 7
@@ -434,8 +434,6 @@ long getThetaLong() {
   private void performIntersect(final Sketch sketchIn) {
     // curCount and input data are nonzero, match against HT
     assert curCount_ > 0 && !empty_;
-    final long[] cacheIn = sketchIn.getCache();
-    final int arrLongsIn = cacheIn.length;
     final long[] hashTable;
     if (wmem_ != null) {
       final int htLen = 1 << lgArrLongs_;
@@ -448,27 +446,16 @@ private void performIntersect(final Sketch sketchIn) {
     final long[] matchSet = new long[ min(curCount_, sketchIn.getRetainedEntries(true)) ];
 
     int matchSetCount = 0;
-    if (sketchIn.isOrdered()) {
-      //ordered compact, which enables early stop
-      for (int i = 0; i < arrLongsIn; i++ ) {
-        final long hashIn = cacheIn[i];
-        //if (hashIn <= 0L) continue;  //<= 0 should not happen
-        if (hashIn >= thetaLong_) {
-          break; //early stop assumes that hashes in input sketch are ordered!
-        }
+    HashIterator it = sketchIn.iterator();
+    while (it.next()) {
+      final long hashIn = it.get();
+      if (hashIn < thetaLong_) {
         final int foundIdx = hashSearch(hashTable, lgArrLongs_, hashIn);
-        if (foundIdx == -1) { continue; }
-        matchSet[matchSetCount++] = hashIn;
-      }
-    }
-    else {
-      //either unordered compact or hash table
-      for (int i = 0; i < arrLongsIn; i++ ) {
-        final long hashIn = cacheIn[i];
-        if (hashIn <= 0L || hashIn >= thetaLong_) { continue; }
-        final int foundIdx = hashSearch(hashTable, lgArrLongs_, hashIn);
-        if (foundIdx == -1) { continue; }
-        matchSet[matchSetCount++] = hashIn;
+        if (foundIdx != -1) {
+          matchSet[matchSetCount++] = hashIn;
+        }
+      } else {
+        if (sketchIn.isOrdered()) { break; } // early stop
       }
     }
     //reduce effective array size to minimum
@@ -515,6 +502,32 @@ private void moveDataToTgt(final long[] arr, final int count) {
     assert tmpCnt == count : "Intersection Count Check: got: " + tmpCnt + ", expected: " + count;
   }
 
+  private void moveDataToTgt(final Sketch sketch) {
+    int count = sketch.getRetainedEntries();
+    int tmpCnt = 0;
+    if (wmem_ != null) { //Off Heap puts directly into mem
+      final int preBytes = CONST_PREAMBLE_LONGS << 3;
+      final int lgArrLongs = lgArrLongs_;
+      final long thetaLong = thetaLong_;
+      HashIterator it = sketch.iterator();
+      while (it.next()) {
+        final long hash = it.get();
+        if (continueCondition(thetaLong, hash)) { continue; }
+        hashInsertOnlyMemory(wmem_, lgArrLongs, hash, preBytes);
+        tmpCnt++;
+      }
+    } else { //On Heap. Assumes HT exists and is large enough
+      HashIterator it = sketch.iterator();
+      while (it.next()) {
+        final long hash = it.get();
+        if (continueCondition(thetaLong_, hash)) { continue; }
+        hashInsertOnly(hashTable_, lgArrLongs_, hash);
+        tmpCnt++;
+      }
+    }
+    assert tmpCnt == count : "Intersection Count Check: got: " + tmpCnt + ", expected: " + count;
+  }
+
   private void hardReset() {
     resetCommon();
     if (wmem_ != null) {
diff --git a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java
index e1d9262e6..ec0bc1268 100644
--- a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java
+++ b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java
@@ -524,4 +524,7 @@ private static void throwNotBigEnough(final long cap, final int required) {
         + ", Required: " + required);
   }
 
+  static int wholeBytesToHoldBits(final int bits) {
+    return (bits >>> 3) + ((bits & 7) > 0 ? 1 : 0);
+  }
 }
diff --git a/src/main/java/org/apache/datasketches/theta/Sketch.java b/src/main/java/org/apache/datasketches/theta/Sketch.java
index 89618bc23..6583e2dbf 100644
--- a/src/main/java/org/apache/datasketches/theta/Sketch.java
+++ b/src/main/java/org/apache/datasketches/theta/Sketch.java
@@ -451,9 +451,8 @@ public String toString(final boolean sketchSummary, final boolean dataDetail, fi
       final boolean hexMode) {
     final StringBuilder sb = new StringBuilder();
 
-    final long[] cache = getCache();
     int nomLongs = 0;
-    int arrLongs = cache.length;
+    int arrLongs = 0;
     float p = 0;
     int rf = 0;
     final boolean updateSketch = this instanceof UpdateSketch;
@@ -473,12 +472,10 @@ public String toString(final boolean sketchSummary, final boolean dataDetail, fi
       final int w = width > 0 ? width : 8; // default is 8 wide
       if (curCount > 0) {
         sb.append("### SKETCH DATA DETAIL");
-        for (int i = 0, j = 0; i < arrLongs; i++ ) {
-          final long h;
-          h = cache[i];
-          if (h <= 0 || h >= thetaLong) {
-            continue;
-          }
+        HashIterator it = iterator();
+        int j = 0;
+        while (it.next()) {
+          final long h = it.get();
           if (j % w == 0) {
             sb.append(LS).append(String.format("   %6d", j + 1));
           }
diff --git a/src/main/java/org/apache/datasketches/theta/UnionImpl.java b/src/main/java/org/apache/datasketches/theta/UnionImpl.java
index bac05de74..4fee6a9c5 100644
--- a/src/main/java/org/apache/datasketches/theta/UnionImpl.java
+++ b/src/main/java/org/apache/datasketches/theta/UnionImpl.java
@@ -320,46 +320,17 @@ public void union(final Sketch sketchIn) {
     }
     //sketchIn is valid and not empty
     ThetaUtil.checkSeedHashes(expectedSeedHash_, sketchIn.getSeedHash());
-    if (sketchIn instanceof SingleItemSketch) {
-      gadget_.hashUpdate(sketchIn.getCache()[0]);
-      return;
-    }
     Sketch.checkSketchAndMemoryFlags(sketchIn);
 
     unionThetaLong_ = min(min(unionThetaLong_, sketchIn.getThetaLong()), gadget_.getThetaLong()); //Theta rule
     unionEmpty_ = false;
-    final int curCountIn = sketchIn.getRetainedEntries(true);
-    if (curCountIn > 0) {
-      if (sketchIn.isOrdered() && (sketchIn instanceof CompactSketch)) { //Use early stop
-        //Ordered, thus compact
-        if (sketchIn.hasMemory()) {
-          final Memory skMem = sketchIn.getMemory();
-          final int preambleLongs = skMem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F;
-          for (int i = 0; i < curCountIn; i++ ) {
-            final int offsetBytes = preambleLongs + i << 3;
-            final long hashIn = skMem.getLong(offsetBytes);
-            if (hashIn >= unionThetaLong_) { break; } // "early stop"
-            gadget_.hashUpdate(hashIn); //backdoor update, hash function is bypassed
-          }
-        }
-        else { //sketchIn is on the Java Heap or has array
-          final long[] cacheIn = sketchIn.getCache(); //not a copy!
-          for (int i = 0; i < curCountIn; i++ ) {
-            final long hashIn = cacheIn[i];
-            if (hashIn >= unionThetaLong_) { break; } // "early stop"
-            gadget_.hashUpdate(hashIn); //backdoor update, hash function is bypassed
-          }
-        }
-      } //End ordered, compact
-      else { //either not-ordered compact or Hash Table form. A HT may have dirty values.
-        final long[] cacheIn = sketchIn.getCache(); //if off-heap this will be a copy
-        final int arrLongs = cacheIn.length;
-        for (int i = 0, c = 0; i < arrLongs && c < curCountIn; i++ ) {
-          final long hashIn = cacheIn[i];
-          if (hashIn <= 0L || hashIn >= unionThetaLong_) { continue; } //rejects dirty values
-          gadget_.hashUpdate(hashIn); //backdoor update, hash function is bypassed
-          c++; //ensures against invalid state inside the incoming sketch
-        }
+    HashIterator it = sketchIn.iterator();
+    while (it.next()) {
+      final long hash = it.get();
+      if (hash < unionThetaLong_ && hash < gadget_.getThetaLong()) {
+        gadget_.hashUpdate(hash); // backdoor update, hash function is bypassed
+      } else {
+        if (sketchIn.isOrdered()) { break; }
       }
     }
     unionThetaLong_ = min(unionThetaLong_, gadget_.getThetaLong()); //Theta rule with gadget
@@ -379,11 +350,8 @@ public void union(final Memory skMem) {
     final int fam = extractFamilyID(skMem);
 
     if (serVer == 4) { // compressed ordered compact
-      // performance can be improved by decompression while performing the union
-      // potentially only partial decompression might be needed
       ThetaUtil.checkSeedHashes(expectedSeedHash_, (short) extractSeedHash(skMem));
-      final CompactSketch csk = CompactSketch.wrap(skMem);
-      union(csk);
+      union(CompactSketch.wrap(skMem));
       return;
     }
     if (serVer == 3) { //The OpenSource sketches (Aug 4, 2015) starts with serVer = 3
@@ -396,16 +364,13 @@ public void union(final Memory skMem) {
     }
     if (serVer == 2) { //older Sketch, which is compact and ordered
       ThetaUtil.checkSeedHashes(expectedSeedHash_, (short)extractSeedHash(skMem));
-      final CompactSketch csk = ForwardCompatibility.heapify2to3(skMem, expectedSeedHash_);
-      union(csk);
+      union(ForwardCompatibility.heapify2to3(skMem, expectedSeedHash_));
       return;
     }
     if (serVer == 1) { //much older Sketch, which is compact and ordered, no seedHash
-      final CompactSketch csk = ForwardCompatibility.heapify1to3(skMem, expectedSeedHash_);
-      union(csk);
+      union(ForwardCompatibility.heapify1to3(skMem, expectedSeedHash_));
       return;
     }
-
     throw new SketchesArgumentException("SerVer is unknown: " + serVer);
   }
 

From a0f1fd6b44e354f62054b5bca76c12c62d47af50 Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Sun, 26 Jan 2025 13:33:41 -0800
Subject: [PATCH 59/88] wrapped compressed sketch

---
 .../theta/DirectCompactCompressedSketch.java  | 134 ++++++++++++++++++
 .../MemoryCompactCompressedHashIterator.java  | 107 ++++++++++++++
 2 files changed, 241 insertions(+)
 create mode 100644 src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java
 create mode 100644 src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java

diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java
new file mode 100644
index 000000000..65b7f877e
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.theta;
+
+import static org.apache.datasketches.theta.PreambleUtil.extractEntryBitsV4;
+import static org.apache.datasketches.theta.PreambleUtil.extractNumEntriesBytesV4;
+import static org.apache.datasketches.theta.PreambleUtil.extractPreLongs;
+import static org.apache.datasketches.theta.PreambleUtil.extractSeedHash;
+import static org.apache.datasketches.theta.PreambleUtil.extractThetaLongV4;
+import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits;
+
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.WritableMemory;
+import org.apache.datasketches.thetacommon.ThetaUtil;
+
+/**
+ * An off-heap (Direct), compact, compressed, read-only sketch. It is not empty, not a single item and ordered.
+ *
+ * 
This sketch can only be associated with a Serialization Version 4 format binary image.

+ *
+ * 
This implementation uses data in a given Memory that is owned and managed by the caller.
+ * This Memory can be off-heap, which if managed properly will greatly reduce the need for
+ * the JVM to perform garbage collection.

+ */
+class DirectCompactCompressedSketch extends DirectCompactSketch {
+  /**
+   * Construct this sketch with the given memory.
+   * @param mem Read-only Memory object with the order bit properly set.
+   */
+  DirectCompactCompressedSketch(final Memory mem) {
+    super(mem);
+  }
+
+  /**
+   * Wraps the given Memory, which must be a SerVer 4 compressed CompactSketch image.
+   * Must check the validity of the Memory before calling. The order bit must be set properly.
+   * @param srcMem See Memory
+   * @param seedHash The update seedHash.
+   * See Seed Hash.
+   * @return this sketch
+   */
+  static DirectCompactCompressedSketch wrapInstance(final Memory srcMem, final short seedHash) {
+    ThetaUtil.checkSeedHashes((short) extractSeedHash(srcMem), seedHash);
+    return new DirectCompactCompressedSketch(srcMem);
+  }
+
+  //Sketch Overrides
+
+  @Override
+  public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem) {
+    if (dstMem != null) {
+      mem_.copyTo(0, dstMem, 0, getCurrentBytes());
+      return new DirectCompactSketch(dstMem);
+    }
+    return CompactSketch.heapify(mem_);
+  }
+
+  @Override
+  public int getCurrentBytes() {
+    final int preLongs = extractPreLongs(mem_);
+    final int entryBits = extractEntryBitsV4(mem_);
+    final int numEntriesBytes = extractNumEntriesBytesV4(mem_);
+    return preLongs * Long.BYTES + numEntriesBytes + wholeBytesToHoldBits(getRetainedEntries() * entryBits);
+  }
+
+  @Override
+  public int getRetainedEntries(final boolean valid) { //compact is always valid
+    final int preLongs = extractPreLongs(mem_);
+    final int numEntriesBytes = extractNumEntriesBytesV4(mem_);
+    int offsetBytes = preLongs > 1 ? 16 : 8;
+    int numEntries = 0;
+    for (int i = 0; i < numEntriesBytes; i++) {
+      numEntries |= Byte.toUnsignedInt(mem_.getByte(offsetBytes++)) << (i << 3);
+    }
+    return numEntries;
+  }
+
+  @Override
+  public long getThetaLong() {
+    final int preLongs = extractPreLongs(mem_);
+    return (preLongs > 1) ? extractThetaLongV4(mem_) : Long.MAX_VALUE;
+  }
+
+  @Override
+  public boolean isEmpty() {
+    return false;
+  }
+
+  @Override
+  public boolean isOrdered() {
+    return true;
+  }
+
+  @Override
+  public HashIterator iterator() {
+    return new MemoryCompactCompressedHashIterator(
+      mem_,
+      (extractPreLongs(mem_) > 1 ? 16 : 8) + extractNumEntriesBytesV4(mem_),
+      extractEntryBitsV4(mem_),
+      getRetainedEntries()
+    );
+  }
+
+  //restricted methods
+
+  @Override
+  long[] getCache() {
+    final int numEntries = getRetainedEntries();
+    final long[] cache = new long[numEntries];
+    int i = 0;
+    HashIterator it = iterator();
+    while (it.next()) {
+      cache[i++] = it.get();
+    }
+    return cache;
+  }
+}
diff --git a/src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java b/src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java
new file mode 100644
index 000000000..b743302a5
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.theta;
+
+import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits;
+
+import org.apache.datasketches.memory.Memory;
+
+/**
+ * @author Lee Rhodes
+ */
+class MemoryCompactCompressedHashIterator implements HashIterator {
+  private Memory mem;
+  private int offset;
+  private int entryBits;
+  private int numEntries;
+  private int index;
+  private long previous;
+  private int offsetBits;
+  private long[] buffer;
+  private byte[] bytes;
+  private boolean isBlockMode;
+  private boolean isFirstUnpack1;
+
+  MemoryCompactCompressedHashIterator(
+      final Memory mem,
+      final int offset,
+      final int entryBits,
+      final int numEntries
+  ) {
+    this.mem = mem;
+    this.offset = offset;
+    this.entryBits = entryBits;
+    this.numEntries = numEntries;
+    index = -1;
+    previous = 0;
+    offsetBits = 0;
+    buffer = new long[8];
+    bytes = new byte[entryBits];
+    isBlockMode = numEntries >= 8;
+    isFirstUnpack1 = true;
+  }
+
+  @Override
+  public long get() {
+    return buffer[index & 7];
+  }
+
+  @Override
+  public boolean next() {
+    if (++index == numEntries) { return false; }
+    if (isBlockMode) {
+      if ((index & 7) == 0) {
+        if (numEntries - index >= 8) {
+          unpack8();
+        } else {
+          isBlockMode = false;
+          unpack1();
+        }
+      }
+    } else {
+      unpack1();
+    }
+    return true;
+  }
+
+  private void unpack1() {
+    if (isFirstUnpack1) {
+      mem.getByteArray(offset, bytes, 0, wholeBytesToHoldBits((numEntries - index) * entryBits));
+      offset = 0;
+      isFirstUnpack1 = false;
+    }
+    final int i = index & 7;
+    BitPacking.unpackBits(buffer, i, entryBits, bytes, offset, offsetBits);
+    offset += (offsetBits + entryBits) >>> 3;
+    offsetBits = (offsetBits + entryBits) & 7;
+    buffer[i] += previous;
+    previous = buffer[i];
+  }
+
+  private void unpack8() {
+    mem.getByteArray(offset, bytes, 0, entryBits);
+    BitPacking.unpackBitsBlock8(buffer, 0, bytes, 0, entryBits);
+    offset += entryBits;
+    for (int i = 0; i < 8; i++) {
+      buffer[i] += previous;
+      previous = buffer[i];
+    }
+  }
+}

From 2698f0d3bed93068514511455a6672549870e67d Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Mon, 27 Jan 2025 09:45:06 -0800
Subject: [PATCH 60/88] restored special treatment of single item sketch

---
 src/main/java/org/apache/datasketches/theta/UnionImpl.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/datasketches/theta/UnionImpl.java b/src/main/java/org/apache/datasketches/theta/UnionImpl.java
index 4fee6a9c5..9b98a9503 100644
--- a/src/main/java/org/apache/datasketches/theta/UnionImpl.java
+++ b/src/main/java/org/apache/datasketches/theta/UnionImpl.java
@@ -22,7 +22,6 @@
 import static java.lang.Math.min;
 import static org.apache.datasketches.theta.PreambleUtil.COMPACT_FLAG_MASK;
 import static org.apache.datasketches.theta.PreambleUtil.ORDERED_FLAG_MASK;
-import static org.apache.datasketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE;
 import static org.apache.datasketches.theta.PreambleUtil.UNION_THETA_LONG;
 import static org.apache.datasketches.theta.PreambleUtil.clearEmpty;
 import static org.apache.datasketches.theta.PreambleUtil.extractCurCount;
@@ -320,6 +319,10 @@ public void union(final Sketch sketchIn) {
     }
     //sketchIn is valid and not empty
     ThetaUtil.checkSeedHashes(expectedSeedHash_, sketchIn.getSeedHash());
+    if (sketchIn instanceof SingleItemSketch) {
+      gadget_.hashUpdate(sketchIn.getCache()[0]);
+      return;
+    }
     Sketch.checkSketchAndMemoryFlags(sketchIn);
 
     unionThetaLong_ = min(min(unionThetaLong_, sketchIn.getThetaLong()), gadget_.getThetaLong()); //Theta rule

From 29895b3d053bb3401cfe3139c5d296c3812f32af Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Mon, 27 Jan 2025 16:13:04 -0800
Subject: [PATCH 61/88] addressed review points

---
 .../org/apache/datasketches/theta/AnotBimpl.java   |  2 +-
 .../theta/DirectCompactCompressedSketch.java       | 14 ++++++++++----
 .../datasketches/theta/IntersectionImpl.java       |  5 +++--
 .../org/apache/datasketches/theta/UnionImpl.java   |  5 +++--
 4 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/theta/AnotBimpl.java b/src/main/java/org/apache/datasketches/theta/AnotBimpl.java
index 6d4c54fd8..cc076fd85 100644
--- a/src/main/java/org/apache/datasketches/theta/AnotBimpl.java
+++ b/src/main/java/org/apache/datasketches/theta/AnotBimpl.java
@@ -200,7 +200,7 @@ private static long[] convertToHashTable(
     final int arrLongs = 1 << lgArrLongs;
     final long[] hashTable = new long[arrLongs];
     checkThetaCorruption(thetaLong);
-    HashIterator it = sketch.iterator();
+    final HashIterator it = sketch.iterator();
     while (it.next()) {
       final long hash = it.get();
       if (continueCondition(thetaLong, hash) ) {
diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java
index 65b7f877e..d7e05ca2e 100644
--- a/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java
@@ -42,7 +42,7 @@
 class DirectCompactCompressedSketch extends DirectCompactSketch {
   /**
    * Construct this sketch with the given memory.
-   * @param mem Read-only Memory object with the order bit properly set.
+   * @param mem Read-only Memory object.
    */
   DirectCompactCompressedSketch(final Memory mem) {
     super(mem);
@@ -50,7 +50,7 @@ class DirectCompactCompressedSketch extends DirectCompactSketch {
 
   /**
    * Wraps the given Memory, which must be a SerVer 4 compressed CompactSketch image.
-   * Must check the validity of the Memory before calling. The order bit must be set properly.
+   * Must check the validity of the Memory before calling.
    * @param srcMem See Memory
    * @param seedHash The update seedHash.
    * See Seed Hash.
@@ -80,11 +80,17 @@ public int getCurrentBytes() {
     return preLongs * Long.BYTES + numEntriesBytes + wholeBytesToHoldBits(getRetainedEntries() * entryBits);
   }
 
+  private static final int START_PACKED_DATA_EXACT_MODE = 8;
+  private static final int START_PACKED_DATA_ESTIMATION_MODE = 16;
+  
   @Override
   public int getRetainedEntries(final boolean valid) { //compact is always valid
-    final int preLongs = extractPreLongs(mem_);
+    // number of entries is stored using variable length encoding
+    // most significant bytes with all zeros are not stored
+    // one byte in the preamble has the number of non-zero bytes used
+    final int preLongs = extractPreLongs(mem_); // if > 1 then the second long has theta
     final int numEntriesBytes = extractNumEntriesBytesV4(mem_);
-    int offsetBytes = preLongs > 1 ? 16 : 8;
+    int offsetBytes = preLongs > 1 ? START_PACKED_DATA_ESTIMATION_MODE : START_PACKED_DATA_EXACT_MODE;
     int numEntries = 0;
     for (int i = 0; i < numEntriesBytes; i++) {
       numEntries |= Byte.toUnsignedInt(mem_.getByte(offsetBytes++)) << (i << 3);
diff --git a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java
index 3404a0b11..b1be73c74 100644
--- a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java
+++ b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java
@@ -446,7 +446,8 @@ private void performIntersect(final Sketch sketchIn) {
     final long[] matchSet = new long[ min(curCount_, sketchIn.getRetainedEntries(true)) ];
 
     int matchSetCount = 0;
-    HashIterator it = sketchIn.iterator();
+    final boolean isOrdered = sketchIn.isOrdered();
+    final HashIterator it = sketchIn.iterator();
     while (it.next()) {
       final long hashIn = it.get();
       if (hashIn < thetaLong_) {
@@ -455,7 +456,7 @@ private void performIntersect(final Sketch sketchIn) {
           matchSet[matchSetCount++] = hashIn;
         }
       } else {
-        if (sketchIn.isOrdered()) { break; } // early stop
+        if (isOrdered) { break; } // early stop
       }
     }
     //reduce effective array size to minimum
diff --git a/src/main/java/org/apache/datasketches/theta/UnionImpl.java b/src/main/java/org/apache/datasketches/theta/UnionImpl.java
index 9b98a9503..d5ae6071a 100644
--- a/src/main/java/org/apache/datasketches/theta/UnionImpl.java
+++ b/src/main/java/org/apache/datasketches/theta/UnionImpl.java
@@ -327,13 +327,14 @@ public void union(final Sketch sketchIn) {
 
     unionThetaLong_ = min(min(unionThetaLong_, sketchIn.getThetaLong()), gadget_.getThetaLong()); //Theta rule
     unionEmpty_ = false;
-    HashIterator it = sketchIn.iterator();
+    final boolean isOrdered = sketchIn.isOrdered();
+    final HashIterator it = sketchIn.iterator();
     while (it.next()) {
       final long hash = it.get();
       if (hash < unionThetaLong_ && hash < gadget_.getThetaLong()) {
         gadget_.hashUpdate(hash); // backdoor update, hash function is bypassed
       } else {
-        if (sketchIn.isOrdered()) { break; }
+        if (isOrdered) { break; }
       }
     }
     unionThetaLong_ = min(unionThetaLong_, gadget_.getThetaLong()); //Theta rule with gadget

From 2f2e5e592dfcc810a552e6917b1fdd7cdf665025 Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Mon, 27 Jan 2025 19:41:40 -0800
Subject: [PATCH 62/88] test equivalence of packing blocks and single values

---
 .../datasketches/theta/BitPackingTest.java    | 65 +++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/src/test/java/org/apache/datasketches/theta/BitPackingTest.java b/src/test/java/org/apache/datasketches/theta/BitPackingTest.java
index a961bffc0..c155bef77 100644
--- a/src/test/java/org/apache/datasketches/theta/BitPackingTest.java
+++ b/src/test/java/org/apache/datasketches/theta/BitPackingTest.java
@@ -40,6 +40,7 @@ public void packUnpackBits() {
           input[i] = value & mask;
           value += Util.INVERSE_GOLDEN_U64;
         }
+
         byte[] bytes = new byte[8 * Long.BYTES];
         int bitOffset = 0;
         int bufOffset = 0;
@@ -57,6 +58,7 @@ public void packUnpackBits() {
           bufOffset += (bitOffset + bits) >>> 3;
           bitOffset = (bitOffset + bits) & 7;
         }
+
         for (int i = 0; i < 8; ++i) {
           assertEquals(output[i], input[i]);
         }
@@ -76,6 +78,7 @@ public void packUnpackBlocks() {
           input[i] = value & mask;
           value += Util.INVERSE_GOLDEN_U64;
         }
+
         byte[] bytes = new byte[8 * Long.BYTES];
         BitPacking.packBitsBlock8(input, 0, bytes, 0, bits);
         if (enablePrinting) { hexDump(bytes); }
@@ -91,6 +94,68 @@ public void packUnpackBlocks() {
     }
   }
 
+  @Test
+  public void packBitsUnpackBlocks() {
+    long value = 0; // arbitrary starting value
+    for (int n = 0; n < 10000; n++) {
+      for (int bits = 1; bits <= 63; bits++) {
+        final long mask = (1 << bits) - 1;
+        long[] input = new long[8];
+        for (int i = 0; i < 8; ++i) {
+          input[i] = value & mask;
+          value += Util.INVERSE_GOLDEN_U64;
+        }
+
+        byte[] bytes = new byte[8 * Long.BYTES];
+        int bitOffset = 0;
+        int bufOffset = 0;
+        for (int i = 0; i < 8; ++i) {
+          BitPacking.packBits(input[i], bits, bytes, bufOffset, bitOffset);
+          bufOffset += (bitOffset + bits) >>> 3;
+          bitOffset = (bitOffset + bits) & 7;
+        }
+
+        long[] output = new long[8];
+        BitPacking.unpackBitsBlock8(output, 0, bytes, 0, bits);
+
+        for (int i = 0; i < 8; ++i) {
+          assertEquals(output[i], input[i]);
+        }
+      }
+    }
+  }
+
+  @Test
+  public void packBlocksUnpackBits() {
+    long value = 123L; // arbitrary starting value
+    for (int n = 0; n < 10000; n++) {
+      for (int bits = 1; bits <= 63; bits++) {
+        final long mask = (1 << bits) - 1;
+        long[] input = new long[8];
+        for (int i = 0; i < 8; ++i) {
+          input[i] = value & mask;
+          value += Util.INVERSE_GOLDEN_U64;
+        }
+
+        byte[] bytes = new byte[8 * Long.BYTES];
+        BitPacking.packBitsBlock8(input, 0, bytes, 0, bits);
+
+        long[] output = new long[8];
+        int bitOffset = 0;
+        int bufOffset = 0;
+        for (int i = 0; i < 8; ++i) {
+          BitPacking.unpackBits(output, i, bits, bytes, bufOffset, bitOffset);
+          bufOffset += (bitOffset + bits) >>> 3;
+          bitOffset = (bitOffset + bits) & 7;
+        }
+
+        for (int i = 0; i < 8; ++i) {
+          assertEquals(output[i], input[i]);
+        }
+      }
+    }
+  }
+
   void hexDump(byte[] bytes) {
     for (int i = 0; i < bytes.length; i++) {
       System.out.print(String.format("%02x ", bytes[i]));

From a0b86ee3cd4f01bd36a5f4e426f4df0d335cd943 Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Wed, 29 Jan 2025 12:05:30 -0800
Subject: [PATCH 63/88] cleanup

---
 .../java/org/apache/datasketches/theta/CompactSketch.java  | 7 +++++--
 .../datasketches/theta/DirectCompactCompressedSketch.java  | 3 ++-
 .../org/apache/datasketches/theta/DirectCompactSketch.java | 5 -----
 .../org/apache/datasketches/theta/HeapCompactSketch.java   | 5 -----
 .../theta/MemoryCompactCompressedHashIterator.java         | 4 ++--
 5 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/theta/CompactSketch.java b/src/main/java/org/apache/datasketches/theta/CompactSketch.java
index 688ad2746..7a366fe0b 100644
--- a/src/main/java/org/apache/datasketches/theta/CompactSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/CompactSketch.java
@@ -188,8 +188,6 @@ private static CompactSketch wrap(final Memory srcMem, final long seed, final bo
     final short seedHash = ThetaUtil.computeSeedHash(seed);
 
     if (serVer == 4) {
-      // not wrapping the compressed format since currently we cannot take advantage of
-      // decompression during iteration because set operations reach into memory directly
       return DirectCompactCompressedSketch.wrapInstance(srcMem,
           enforceSeed ? seedHash : (short) extractSeedHash(srcMem));
     }
@@ -251,6 +249,11 @@ public boolean isCompact() {
     return true;
   }
 
+  @Override
+  public double getEstimate() {
+    return Sketch.estimate(getThetaLong(), getRetainedEntries());
+  }
+
   /**
    * gets the sketch as a compressed byte array
    * @return the sketch as a compressed byte array
diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java
index d7e05ca2e..6c83add87 100644
--- a/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java
@@ -118,7 +118,8 @@ public boolean isOrdered() {
   public HashIterator iterator() {
     return new MemoryCompactCompressedHashIterator(
       mem_,
-      (extractPreLongs(mem_) > 1 ? 16 : 8) + extractNumEntriesBytesV4(mem_),
+      (extractPreLongs(mem_) > 1 ? START_PACKED_DATA_ESTIMATION_MODE : START_PACKED_DATA_EXACT_MODE)
+        + extractNumEntriesBytesV4(mem_),
       extractEntryBitsV4(mem_),
       getRetainedEntries()
     );
diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java
index 1714d2161..baedad179 100644
--- a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java
@@ -84,11 +84,6 @@ public int getCurrentBytes() {
     return (preLongs + curCount) << 3;
   }
 
-  @Override
-  public double getEstimate() {
-    return Sketch.estimate(getThetaLong(), getRetainedEntries());
-  }
-
   @Override
   public int getRetainedEntries(final boolean valid) { //compact is always valid
     if (otherCheckForSingleItem(mem_)) { return 1; }
diff --git a/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java b/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java
index f394e9303..2572ce5d5 100644
--- a/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java
@@ -87,11 +87,6 @@ public int getCurrentBytes() {
     return (preLongs_ + curCount_) << 3;
   }
 
-  @Override
-  public double getEstimate() {
-    return Sketch.estimate(thetaLong_, curCount_);
-  }
-
   @Override
   public int getRetainedEntries(final boolean valid) {
     return curCount_;
diff --git a/src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java b/src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java
index b743302a5..d5f37de96 100644
--- a/src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java
+++ b/src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java
@@ -23,8 +23,8 @@
 
 import org.apache.datasketches.memory.Memory;
 
-/**
- * @author Lee Rhodes
+/*
+ * This is to uncompress serial version 4 sketch incrementally
  */
 class MemoryCompactCompressedHashIterator implements HashIterator {
   private Memory mem;

From bde0b6f2689ea671b7a2f72c1ecdf028565e3f9c Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Tue, 4 Feb 2025 16:15:20 -0800
Subject: [PATCH 64/88] wrap(byte[])

---
 .../BytesCompactCompressedHashIterator.java   |  93 ++++++++++
 .../theta/BytesCompactHashIterator.java       |  53 ++++++
 .../datasketches/theta/CompactSketch.java     |  54 ++++++
 .../theta/DirectCompactSketch.java            |   4 +-
 .../theta/WrappedCompactCompressedSketch.java | 111 ++++++++++++
 .../theta/WrappedCompactSketch.java           | 165 ++++++++++++++++++
 .../datasketches/theta/CompactSketchTest.java |  34 ++++
 7 files changed, 512 insertions(+), 2 deletions(-)
 create mode 100644 src/main/java/org/apache/datasketches/theta/BytesCompactCompressedHashIterator.java
 create mode 100644 src/main/java/org/apache/datasketches/theta/BytesCompactHashIterator.java
 create mode 100644 src/main/java/org/apache/datasketches/theta/WrappedCompactCompressedSketch.java
 create mode 100644 src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java

diff --git a/src/main/java/org/apache/datasketches/theta/BytesCompactCompressedHashIterator.java b/src/main/java/org/apache/datasketches/theta/BytesCompactCompressedHashIterator.java
new file mode 100644
index 000000000..97792da26
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/theta/BytesCompactCompressedHashIterator.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.theta;
+
+/*
+ * This is to uncompress serial version 4 sketch incrementally
+ */
+class BytesCompactCompressedHashIterator implements HashIterator {
+  private byte[] bytes;
+  private int offset;
+  private int entryBits;
+  private int numEntries;
+  private int index;
+  private long previous;
+  private int offsetBits;
+  private long[] buffer;
+  private boolean isBlockMode;
+
+  BytesCompactCompressedHashIterator(
+      final byte[] bytes,
+      final int offset,
+      final int entryBits,
+      final int numEntries
+  ) {
+    this.bytes = bytes;
+    this.offset = offset;
+    this.entryBits = entryBits;
+    this.numEntries = numEntries;
+    index = -1;
+    previous = 0;
+    offsetBits = 0;
+    buffer = new long[8];
+    isBlockMode = numEntries >= 8;
+  }
+
+  @Override
+  public long get() {
+    return buffer[index & 7];
+  }
+
+  @Override
+  public boolean next() {
+    if (++index == numEntries) { return false; }
+    if (isBlockMode) {
+      if ((index & 7) == 0) {
+        if (numEntries - index >= 8) {
+          unpack8();
+        } else {
+          isBlockMode = false;
+          unpack1();
+        }
+      }
+    } else {
+      unpack1();
+    }
+    return true;
+  }
+
+  private void unpack1() {
+    final int i = index & 7;
+    BitPacking.unpackBits(buffer, i, entryBits, bytes, offset, offsetBits);
+    offset += (offsetBits + entryBits) >>> 3;
+    offsetBits = (offsetBits + entryBits) & 7;
+    buffer[i] += previous;
+    previous = buffer[i];
+  }
+
+  private void unpack8() {
+    BitPacking.unpackBitsBlock8(buffer, 0, bytes, offset, entryBits);
+    offset += entryBits;
+    for (int i = 0; i < 8; i++) {
+      buffer[i] += previous;
+      previous = buffer[i];
+    }
+  }
+}
diff --git a/src/main/java/org/apache/datasketches/theta/BytesCompactHashIterator.java b/src/main/java/org/apache/datasketches/theta/BytesCompactHashIterator.java
new file mode 100644
index 000000000..20e21da11
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/theta/BytesCompactHashIterator.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.theta;
+
+import org.apache.datasketches.common.ByteArrayUtil;
+
+/*
+ * This is to iterate over serial version 3 sketch representation
+ */
+class BytesCompactHashIterator implements HashIterator {
+  final private byte[] bytes;
+  final private int offset;
+  final private int numEntries;
+  private int index;
+
+  BytesCompactHashIterator(
+      final byte[] bytes,
+      final int offset,
+      final int numEntries
+  ) {
+    this.bytes = bytes;
+    this.offset = offset;
+    this.numEntries = numEntries;
+    index = -1;
+  }
+
+  @Override
+  public long get() {
+    return ByteArrayUtil.getLongLE(bytes, offset + index * Long.BYTES);
+  }
+
+  @Override
+  public boolean next() {
+    return ++index < numEntries;
+ }
+}
diff --git a/src/main/java/org/apache/datasketches/theta/CompactSketch.java b/src/main/java/org/apache/datasketches/theta/CompactSketch.java
index 7a366fe0b..b64314168 100644
--- a/src/main/java/org/apache/datasketches/theta/CompactSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/CompactSketch.java
@@ -19,11 +19,15 @@
 
 package org.apache.datasketches.theta;
 
+import static org.apache.datasketches.common.ByteArrayUtil.getShortLE;
 import static org.apache.datasketches.common.Family.idToFamily;
 import static org.apache.datasketches.theta.PreambleUtil.COMPACT_FLAG_MASK;
 import static org.apache.datasketches.theta.PreambleUtil.EMPTY_FLAG_MASK;
+import static org.apache.datasketches.theta.PreambleUtil.FLAGS_BYTE;
 import static org.apache.datasketches.theta.PreambleUtil.ORDERED_FLAG_MASK;
+import static org.apache.datasketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE;
 import static org.apache.datasketches.theta.PreambleUtil.READ_ONLY_FLAG_MASK;
+import static org.apache.datasketches.theta.PreambleUtil.SEED_HASH_SHORT;
 import static org.apache.datasketches.theta.PreambleUtil.extractFamilyID;
 import static org.apache.datasketches.theta.PreambleUtil.extractFlags;
 import static org.apache.datasketches.theta.PreambleUtil.extractPreLongs;
@@ -224,6 +228,56 @@ else if (serVer == 2) {
         "Corrupted: Serialization Version " + serVer + " not recognized.");
   }
 
+  public static CompactSketch wrap(final byte[] bytes) {
+    return wrap(bytes, ThetaUtil.DEFAULT_UPDATE_SEED, false);
+  }
+  
+  public static CompactSketch wrap(final byte[] bytes, final long expectedSeed) {
+    return wrap(bytes, expectedSeed, true);
+  }
+  
+  private static CompactSketch wrap(final byte[] bytes, final long seed, final boolean enforceSeed) {
+    final int serVer = bytes[PreambleUtil.SER_VER_BYTE];
+    final int familyId = bytes[PreambleUtil.FAMILY_BYTE];
+    final Family family = Family.idToFamily(familyId);
+    if (family != Family.COMPACT) {
+      throw new IllegalArgumentException("Corrupted: " + family + " is not Compact!");
+    }
+    final short seedHash = ThetaUtil.computeSeedHash(seed);
+    if (serVer == 4) {
+      return WrappedCompactCompressedSketch.wrapInstance(bytes, seedHash);
+    } else if (serVer == 3) {
+      final int flags = bytes[FLAGS_BYTE];
+      if ((flags & EMPTY_FLAG_MASK) > 0) {
+        return EmptyCompactSketch.getHeapInstance(Memory.wrap(bytes));
+      }
+      final int preLongs = bytes[PREAMBLE_LONGS_BYTE];
+      if (otherCheckForSingleItem(preLongs, serVer, familyId, flags)) {
+        return SingleItemSketch.heapify(Memory.wrap(bytes), enforceSeed ? seedHash : getShortLE(bytes, SEED_HASH_SHORT));
+      }
+      //not empty & not singleItem
+      final boolean compactFlag = (flags & COMPACT_FLAG_MASK) > 0;
+      if (!compactFlag) {
+        throw new SketchesArgumentException(
+            "Corrupted: COMPACT family sketch image must have compact flag set");
+      }
+      final boolean readOnly = (flags & READ_ONLY_FLAG_MASK) > 0;
+      if (!readOnly) {
+        throw new SketchesArgumentException(
+            "Corrupted: COMPACT family sketch image must have Read-Only flag set");
+      }
+      return WrappedCompactSketch.wrapInstance(bytes,
+          enforceSeed ? seedHash : getShortLE(bytes, SEED_HASH_SHORT));
+    } else if (serVer == 1) {
+      return ForwardCompatibility.heapify1to3(Memory.wrap(bytes), seedHash);
+    } else if (serVer == 2) {
+      return ForwardCompatibility.heapify2to3(Memory.wrap(bytes),
+          enforceSeed ? seedHash : getShortLE(bytes, SEED_HASH_SHORT));
+    }
+    throw new SketchesArgumentException(
+        "Corrupted: Serialization Version " + serVer + " not recognized.");
+  }
+
   //Sketch Overrides
 
   @Override
diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java
index baedad179..15b03311b 100644
--- a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java
@@ -35,7 +35,7 @@
 
 /**
  * An off-heap (Direct), compact, read-only sketch. The internal hash array can be either ordered
- * or unordered.
+ * or unordered. It is not empty, not a single item.
  *
  * 
This sketch can only be associated with a Serialization Version 3 format binary image.

  *
@@ -57,7 +57,7 @@ class DirectCompactSketch extends CompactSketch {
   }
 
   /**
-   * Wraps the given Memory, which must be a SerVer 3, ordered, CompactSketch image.
+   * Wraps the given Memory, which must be a SerVer 3, CompactSketch image.
    * Must check the validity of the Memory before calling. The order bit must be set properly.
    * @param srcMem See Memory
    * @param seedHash The update seedHash.
diff --git a/src/main/java/org/apache/datasketches/theta/WrappedCompactCompressedSketch.java b/src/main/java/org/apache/datasketches/theta/WrappedCompactCompressedSketch.java
new file mode 100644
index 000000000..170c98cad
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/theta/WrappedCompactCompressedSketch.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.theta;
+
+import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits;
+import static org.apache.datasketches.theta.PreambleUtil.ENTRY_BITS_BYTE_V4;
+import static org.apache.datasketches.theta.PreambleUtil.NUM_ENTRIES_BYTES_BYTE_V4;
+import static org.apache.datasketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE;
+
+import org.apache.datasketches.common.ByteArrayUtil;
+import org.apache.datasketches.thetacommon.ThetaUtil;
+
+/**
+ * Wrapper around a serialized compact compressed read-only sketch. It is not empty, not a single item.
+ *
+ * 
This sketch can only be associated with a Serialization Version 4 format binary image.

+ */
+class WrappedCompactCompressedSketch extends WrappedCompactSketch {
+  
+  /**
+   * Construct this sketch with the given bytes.
+   * @param bytes containing serialized compact compressed sketch.
+   */
+  WrappedCompactCompressedSketch(final byte[] bytes) {
+    super(bytes);
+  }
+
+  /**
+   * Wraps the given bytes, which must be a SerVer 4 compressed CompactSketch image.
+   * @param bytes representation of serialized compressed compact sketch.
+   * @param seedHash The update seedHash.
+   * See Seed Hash.
+   * @return this sketch
+   */
+  static WrappedCompactCompressedSketch wrapInstance(final byte[] bytes, final short seedHash) {
+    ThetaUtil.checkSeedHashes(ByteArrayUtil.getShortLE(bytes, PreambleUtil.SEED_HASH_SHORT), seedHash);
+    return new WrappedCompactCompressedSketch(bytes);
+  }
+
+  //Sketch Overrides
+
+  @Override
+  public int getCurrentBytes() {
+    final int preLongs = bytes_[PREAMBLE_LONGS_BYTE];
+    final int entryBits = bytes_[ENTRY_BITS_BYTE_V4];
+    final int numEntriesBytes = bytes_[NUM_ENTRIES_BYTES_BYTE_V4];
+    return preLongs * Long.BYTES + numEntriesBytes + wholeBytesToHoldBits(getRetainedEntries() * entryBits);
+  }
+
+  private static final int START_PACKED_DATA_EXACT_MODE = 8;
+  private static final int START_PACKED_DATA_ESTIMATION_MODE = 16;
+  
+  @Override
+  public int getRetainedEntries(final boolean valid) { //compact is always valid
+    // number of entries is stored using variable length encoding
+    // most significant bytes with all zeros are not stored
+    // one byte in the preamble has the number of non-zero bytes used
+    final int preLongs = bytes_[PREAMBLE_LONGS_BYTE]; // if > 1 then the second long has theta
+    final int numEntriesBytes = bytes_[NUM_ENTRIES_BYTES_BYTE_V4];
+    int offsetBytes = preLongs > 1 ? START_PACKED_DATA_ESTIMATION_MODE : START_PACKED_DATA_EXACT_MODE;
+    int numEntries = 0;
+    for (int i = 0; i < numEntriesBytes; i++) {
+      numEntries |= Byte.toUnsignedInt(bytes_[offsetBytes++]) << (i << 3);
+    }
+    return numEntries;
+  }
+
+  @Override
+  public long getThetaLong() {
+    final int preLongs = bytes_[PREAMBLE_LONGS_BYTE];
+    return (preLongs > 1) ? ByteArrayUtil.getLongLE(bytes_, 8) : Long.MAX_VALUE;
+  }
+
+  @Override
+  public boolean isEmpty() {
+    return false;
+  }
+
+  @Override
+  public boolean isOrdered() {
+    return true;
+  }
+
+  @Override
+  public HashIterator iterator() {
+    return new BytesCompactCompressedHashIterator(
+      bytes_,
+      (bytes_[PREAMBLE_LONGS_BYTE] > 1 ? START_PACKED_DATA_ESTIMATION_MODE : START_PACKED_DATA_EXACT_MODE)
+        + bytes_[NUM_ENTRIES_BYTES_BYTE_V4],
+      bytes_[ENTRY_BITS_BYTE_V4],
+      getRetainedEntries()
+    );
+  }
+}
diff --git a/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java b/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java
new file mode 100644
index 000000000..b1073159b
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.theta;
+
+import static org.apache.datasketches.common.ByteArrayUtil.getIntLE;
+import static org.apache.datasketches.common.ByteArrayUtil.getLongLE;
+import static org.apache.datasketches.common.ByteArrayUtil.getShortLE;
+import static org.apache.datasketches.theta.CompactOperations.memoryToCompact;
+import static org.apache.datasketches.theta.PreambleUtil.EMPTY_FLAG_MASK;
+import static org.apache.datasketches.theta.PreambleUtil.ORDERED_FLAG_MASK;
+import static org.apache.datasketches.theta.PreambleUtil.FLAGS_BYTE;
+import static org.apache.datasketches.theta.PreambleUtil.RETAINED_ENTRIES_INT;
+import static org.apache.datasketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE;
+import static org.apache.datasketches.theta.PreambleUtil.THETA_LONG;
+import static org.apache.datasketches.theta.PreambleUtil.SEED_HASH_SHORT;
+
+import java.util.Arrays;
+
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.WritableMemory;
+import org.apache.datasketches.thetacommon.ThetaUtil;
+
+/**
+ * Wrapper around a serialized compact read-only sketch. It is not empty, not a single item.
+ *
+ * 
This sketch can only be associated with a Serialization Version 3 format binary image.

+ */
+class WrappedCompactSketch extends CompactSketch {
+  final byte[] bytes_;
+
+  /**
+   * Construct this sketch with the given bytes.
+   * @param bytes containing serialized compact sketch.
+   */
+  WrappedCompactSketch(final byte[] bytes) {
+    bytes_ = bytes;
+  }
+
+  /**
+   * Wraps the given Memory, which must be a SerVer 3 CompactSketch image.
+   * @param bytes representation of serialized compressed compact sketch.
+   * @param seedHash The update seedHash.
+   * See Seed Hash.
+   * @return this sketch
+   */
+  static WrappedCompactSketch wrapInstance(final byte[] bytes, final short seedHash) {
+    ThetaUtil.checkSeedHashes(getShortLE(bytes, SEED_HASH_SHORT), seedHash);
+    return new WrappedCompactSketch(bytes);
+  }
+
+  //Sketch Overrides
+
+  @Override
+  public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem) {
+    return memoryToCompact(Memory.wrap(bytes_), dstOrdered, dstMem);
+  }
+
+  @Override
+  public int getCurrentBytes() {
+    final int preLongs = bytes_[PreambleUtil.PREAMBLE_LONGS_BYTE];
+    final int numEntries = (preLongs == 1) ? 0 : getIntLE(bytes_, RETAINED_ENTRIES_INT);
+    return (preLongs + numEntries) << 3;
+  }
+
+  @Override
+  public int getRetainedEntries(final boolean valid) { //compact is always valid
+    final int preLongs = bytes_[PREAMBLE_LONGS_BYTE];
+    return (preLongs == 1) ? 0 : getIntLE(bytes_, RETAINED_ENTRIES_INT);
+  }
+
+  @Override
+  public long getThetaLong() {
+    final int preLongs = bytes_[PREAMBLE_LONGS_BYTE];
+    return (preLongs > 2) ? getLongLE(bytes_, THETA_LONG) : Long.MAX_VALUE;
+  }
+
+  @Override
+  public boolean hasMemory() {
+    return false;
+  }
+
+  @Override
+  public boolean isDirect() {
+    return false;
+  }
+
+  @Override
+  public boolean isEmpty() {
+    return (bytes_[FLAGS_BYTE] & EMPTY_FLAG_MASK) > 0;
+  }
+
+  @Override
+  public boolean isOrdered() {
+    return (bytes_[FLAGS_BYTE] & ORDERED_FLAG_MASK) > 0;
+  }
+
+  @Override
+  public boolean isSameResource(final Memory that) {
+    return false;
+  }
+
+  @Override
+  public HashIterator iterator() {
+    return new BytesCompactHashIterator(
+      bytes_,
+      bytes_[PREAMBLE_LONGS_BYTE] << 3,
+      getRetainedEntries()
+    );
+  }
+
+  @Override
+  public byte[] toByteArray() {
+    return Arrays.copyOf(bytes_, getCurrentBytes());
+  }
+
+  //restricted methods
+
+  @Override
+  long[] getCache() {
+    final long[] cache = new long[getRetainedEntries()];
+    int i = 0;
+    HashIterator it = iterator();
+    while (it.next()) {
+      cache[i++] = it.get();
+    }
+    return cache;
+  }
+
+  @Override
+  int getCompactPreambleLongs() {
+    return bytes_[PREAMBLE_LONGS_BYTE];
+  }
+
+  @Override
+  int getCurrentPreambleLongs() {
+    return bytes_[PREAMBLE_LONGS_BYTE];
+  }
+
+  @Override
+  Memory getMemory() {
+    return null;
+  }
+
+  @Override
+  short getSeedHash() {
+    return getShortLE(bytes_, SEED_HASH_SHORT);
+  }
+}
diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
index e0f79087b..0b49f2389 100644
--- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
@@ -587,6 +587,40 @@ public void serializeDeserializeDirectV4() {
     }
   }
 
+  @Test
+  public void serializeWrapBytesV3() {
+    UpdateSketch sk = Sketches.updateSketchBuilder().build();
+    for (int i = 0; i < 10000; i++) {
+      sk.update(i);
+    }
+    CompactSketch cs1 = sk.compact();
+    byte[] bytes = cs1.toByteArray();
+    CompactSketch cs2 = new WrappedCompactSketch(bytes);
+    assertEquals(cs1.getRetainedEntries(), cs2.getRetainedEntries());
+    HashIterator it1 = cs1.iterator();
+    HashIterator it2 = cs2.iterator();
+    while (it1.next() && it2.next()) {
+      assertEquals(it2.get(), it2.get());
+    }
+  }
+
+  @Test
+  public void serializeWrapBytesV4() {
+    UpdateSketch sk = Sketches.updateSketchBuilder().build();
+    for (int i = 0; i < 10000; i++) {
+      sk.update(i);
+    }
+    CompactSketch cs1 = sk.compact();
+    byte[] bytes = cs1.toByteArrayCompressed();
+    CompactSketch cs2 = new WrappedCompactCompressedSketch(bytes);
+    assertEquals(cs1.getRetainedEntries(), cs2.getRetainedEntries());
+    HashIterator it1 = cs1.iterator();
+    HashIterator it2 = cs2.iterator();
+    while (it1.next() && it2.next()) {
+      assertEquals(it2.get(), it2.get());
+    }
+  }
+
   private static class State {
     String classType = null;
     int count = 0;

From 8b13d168e917b47db13143cda669c80dd2928047 Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Tue, 4 Feb 2025 18:25:44 -0800
Subject: [PATCH 65/88] test toByteArray()

---
 .../datasketches/theta/CompactSketchTest.java | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
index 0b49f2389..05a0e8eb5 100644
--- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
@@ -589,36 +589,38 @@ public void serializeDeserializeDirectV4() {
 
   @Test
   public void serializeWrapBytesV3() {
-    UpdateSketch sk = Sketches.updateSketchBuilder().build();
+    final UpdateSketch sk = Sketches.updateSketchBuilder().build();
     for (int i = 0; i < 10000; i++) {
       sk.update(i);
     }
-    CompactSketch cs1 = sk.compact();
-    byte[] bytes = cs1.toByteArray();
-    CompactSketch cs2 = new WrappedCompactSketch(bytes);
+    final CompactSketch cs1 = sk.compact();
+    final byte[] bytes = cs1.toByteArray();
+    final CompactSketch cs2 = new WrappedCompactSketch(bytes);
     assertEquals(cs1.getRetainedEntries(), cs2.getRetainedEntries());
-    HashIterator it1 = cs1.iterator();
-    HashIterator it2 = cs2.iterator();
+    final HashIterator it1 = cs1.iterator();
+    final HashIterator it2 = cs2.iterator();
     while (it1.next() && it2.next()) {
       assertEquals(it2.get(), it2.get());
     }
+    assertEquals(bytes, cs2.toByteArray());
   }
 
   @Test
   public void serializeWrapBytesV4() {
-    UpdateSketch sk = Sketches.updateSketchBuilder().build();
+    final UpdateSketch sk = Sketches.updateSketchBuilder().build();
     for (int i = 0; i < 10000; i++) {
       sk.update(i);
     }
-    CompactSketch cs1 = sk.compact();
-    byte[] bytes = cs1.toByteArrayCompressed();
-    CompactSketch cs2 = new WrappedCompactCompressedSketch(bytes);
+    final CompactSketch cs1 = sk.compact();
+    final byte[] bytes = cs1.toByteArrayCompressed();
+    final CompactSketch cs2 = new WrappedCompactCompressedSketch(bytes);
     assertEquals(cs1.getRetainedEntries(), cs2.getRetainedEntries());
-    HashIterator it1 = cs1.iterator();
-    HashIterator it2 = cs2.iterator();
+    final HashIterator it1 = cs1.iterator();
+    final HashIterator it2 = cs2.iterator();
     while (it1.next() && it2.next()) {
       assertEquals(it2.get(), it2.get());
     }
+    assertEquals(bytes, cs2.toByteArray());
   }
 
   private static class State {

From 7596110166bac05017f406e9800ecd375bacfc5d Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Thu, 6 Feb 2025 14:03:04 -0800
Subject: [PATCH 66/88] Fix misc SuppressWarnings

---
 .../org/apache/datasketches/quantilescommon/QuantilesAPI.java   | 2 +-
 .../datasketches/thetacommon/SetOperationCornerCases.java       | 2 +-
 .../org/apache/datasketches/tuple/SerializerDeserializer.java   | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java
index a082fc27a..471ee5570 100644
--- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java
+++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java
@@ -202,7 +202,7 @@
  * @author Kevin Lang
  * @author Alexander Saydakov
  */
-@SuppressWarnings("javadoc")
+
 public interface QuantilesAPI {
 
   /** The sketch must not be empty for this operation. */
diff --git a/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java b/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java
index d9fda48bb..2457b37fe 100644
--- a/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java
+++ b/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java
@@ -28,7 +28,7 @@
  * Simplifies and speeds up set operations by resolving specific corner cases.
  * @author Lee Rhodes
  */
-@SuppressWarnings("javadoc")
+
 public class SetOperationCornerCases {
   private static final long MAX = Long.MAX_VALUE;
 
diff --git a/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java b/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java
index 9b0ca33cb..a30d47edf 100644
--- a/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java
+++ b/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java
@@ -31,7 +31,6 @@ public final class SerializerDeserializer {
   /**
    * Defines the sketch classes that this SerializerDeserializer can handle.
    */
-  @SuppressWarnings("javadoc")
   public static enum SketchType {
     /** QuickSelectSketch */
     QuickSelectSketch,

From 3034a347162efdb2fb75c18b231cf0c018920bea Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Sat, 8 Feb 2025 20:48:08 -0800
Subject: [PATCH 67/88] Changes to MurmurHash3v3 to *3v4

---
 ...ash3v3Test.java => MurmurHash3v4Test.java} | 72 +++++++++----------
 1 file changed, 36 insertions(+), 36 deletions(-)
 rename src/test/java/org/apache/datasketches/hash/{MurmurHash3v3Test.java => MurmurHash3v4Test.java} (83%)

diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3v3Test.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3v4Test.java
similarity index 83%
rename from src/test/java/org/apache/datasketches/hash/MurmurHash3v3Test.java
rename to src/test/java/org/apache/datasketches/hash/MurmurHash3v4Test.java
index 8699a091a..6cd4d624f 100644
--- a/src/test/java/org/apache/datasketches/hash/MurmurHash3v3Test.java
+++ b/src/test/java/org/apache/datasketches/hash/MurmurHash3v4Test.java
@@ -28,13 +28,13 @@
 import org.testng.annotations.Test;
 
 import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.memory.internal.MurmurHash3v3;
+import org.apache.datasketches.memory.internal.MurmurHash3v4;
 import org.apache.datasketches.memory.WritableMemory;
 
 /**
  * @author Lee Rhodes
  */
-public class MurmurHash3v3Test {
+public class MurmurHash3v4Test {
   private Random rand = new Random();
   private static final int trials = 1 << 20;
 
@@ -154,33 +154,33 @@ private static final long[] hashV1(byte[] key, long seed) {
   }
 
   private static final long[] hashV2(long[] key, long seed) {
-    return MurmurHash3v3.hash(key, seed);
+    return MurmurHash3v4.hash(key, seed);
   }
 
   private static final long[] hashV2(int[] key2, long seed) {
-    return MurmurHash3v3.hash(key2, seed);
+    return MurmurHash3v4.hash(key2, seed);
   }
 
   private static final long[] hashV2(char[] key, long seed) {
-    return MurmurHash3v3.hash(key, seed);
+    return MurmurHash3v4.hash(key, seed);
   }
 
   private static final long[] hashV2(byte[] key, long seed) {
-    return MurmurHash3v3.hash(key, seed);
+    return MurmurHash3v4.hash(key, seed);
   }
 
   //V2 single primitives
 
   private static final long[] hashV2(long key, long seed, long[] out) {
-    return MurmurHash3v3.hash(key, seed, out);
+    return MurmurHash3v4.hash(key, seed, out);
   }
 
 //  private static final long[] hashV2(double key, long seed, long[] out) {
-//    return MurmurHash3v3.hash(key, seed, out);
+//    return MurmurHash3v4.hash(key, seed, out);
 //  }
 
 //  private static final long[] hashV2(String key, long seed, long[] out) {
-//    return MurmurHash3v3.hash(key, seed, out);
+//    return MurmurHash3v4.hash(key, seed, out);
 //  }
 
 
@@ -197,9 +197,9 @@ public void offsetChecks() {
     WritableMemory wmem = WritableMemory.allocate(cap);
     for (int i = 0; i < cap; i++) { wmem.putByte(i, (byte)(-128 + i)); }
 
-    for (int offset = 0; offset < 16; offset++) {
+    for (int offset = 3; offset < 16; offset++) {
       int arrLen = cap - offset;
-      hash1 = MurmurHash3v3.hash(wmem, offset, arrLen, seed, hash1);
+      hash1 = MurmurHash3v4.hash(wmem, offset, arrLen, seed, hash1);
       byte[] byteArr2 = new byte[arrLen];
       wmem.getByteArray(offset, byteArr2, 0, arrLen);
       hash2 = MurmurHash3.hash(byteArr2, seed);
@@ -222,8 +222,8 @@ public void byteArrChecks() {
       for (int i = 0; i < j; i++) { wmem.putByte(i, (byte) (-128 + i)); }
 
       long[] hash1 = MurmurHash3.hash(in, 0);
-      hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2);
-      long[] hash3 = MurmurHash3v3.hash(in, seed);
+      hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2);
+      long[] hash3 = MurmurHash3v4.hash(in, seed);
 
       assertEquals(hash1, hash2);
       assertEquals(hash1, hash3);
@@ -246,8 +246,8 @@ public void charArrChecks() {
       for (int i = 0; i < j; i++) { wmem.putInt(i, i); }
 
       long[] hash1 = MurmurHash3.hash(in, 0);
-      hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2);
-      long[] hash3 = MurmurHash3v3.hash(in, seed);
+      hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2);
+      long[] hash3 = MurmurHash3v4.hash(in, seed);
 
       assertEquals(hash1, hash2);
       assertEquals(hash1, hash3);
@@ -270,8 +270,8 @@ public void intArrChecks() {
       for (int i = 0; i < j; i++) { wmem.putInt(i, i); }
 
       long[] hash1 = MurmurHash3.hash(in, 0);
-      hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2);
-      long[] hash3 = MurmurHash3v3.hash(in, seed);
+      hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2);
+      long[] hash3 = MurmurHash3v4.hash(in, seed);
 
       assertEquals(hash1, hash2);
       assertEquals(hash1, hash3);
@@ -294,8 +294,8 @@ public void longArrChecks() {
       for (int i = 0; i < j; i++) { wmem.putLong(i, i); }
 
       long[] hash1 = MurmurHash3.hash(in, 0);
-      hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2);
-      long[] hash3 = MurmurHash3v3.hash(in, seed);
+      hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2);
+      long[] hash3 = MurmurHash3v4.hash(in, seed);
 
       assertEquals(hash1, hash2);
       assertEquals(hash1, hash3);
@@ -313,8 +313,8 @@ public void longCheck() {
     WritableMemory wmem = WritableMemory.writableWrap(in);
 
     long[] hash1 = MurmurHash3.hash(in, 0);
-    hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2);
-    long[] hash3 = MurmurHash3v3.hash(in, seed);
+    hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2);
+    long[] hash3 = MurmurHash3v4.hash(in, seed);
 
     assertEquals(hash1, hash2);
     assertEquals(hash1, hash3);
@@ -325,57 +325,57 @@ public void checkEmptiesNulls() {
     long seed = 123;
     long[] hashOut = new long[2];
     try {
-      MurmurHash3v3.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut);  //mem empty
+      MurmurHash3v4.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut);  //mem empty
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       String s = "";
-      MurmurHash3v3.hash(s, seed, hashOut); //string empty
+      MurmurHash3v4.hash(s, seed, hashOut); //string empty
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       String s = null;
-      MurmurHash3v3.hash(s, seed, hashOut); //string null
+      MurmurHash3v4.hash(s, seed, hashOut); //string null
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       byte[] barr = new byte[0];
-      MurmurHash3v3.hash(barr, seed); //byte[] empty
+      MurmurHash3v4.hash(barr, seed); //byte[] empty
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       byte[] barr = null;
-      MurmurHash3v3.hash(barr, seed); //byte[] null
+      MurmurHash3v4.hash(barr, seed); //byte[] null
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       char[] carr = new char[0];
-      MurmurHash3v3.hash(carr, seed); //char[] empty
+      MurmurHash3v4.hash(carr, seed); //char[] empty
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       char[] carr = null;
-      MurmurHash3v3.hash(carr, seed); //char[] null
+      MurmurHash3v4.hash(carr, seed); //char[] null
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       int[] iarr = new int[0];
-      MurmurHash3v3.hash(iarr, seed); //int[] empty
+      MurmurHash3v4.hash(iarr, seed); //int[] empty
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       int[] iarr = null;
-      MurmurHash3v3.hash(iarr, seed); //int[] null
+      MurmurHash3v4.hash(iarr, seed); //int[] null
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       long[] larr = new long[0];
-      MurmurHash3v3.hash(larr, seed); //long[] empty
+      MurmurHash3v4.hash(larr, seed); //long[] empty
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       long[] larr = null;
-      MurmurHash3v3.hash(larr, seed); //long[] null
+      MurmurHash3v4.hash(larr, seed); //long[] null
       fail();
     } catch (final IllegalArgumentException e) { } //OK
   }
@@ -385,9 +385,9 @@ public void checkStringLong() {
     long seed = 123;
     long[] hashOut = new long[2];
     String s = "123";
-    assertTrue(MurmurHash3v3.hash(s, seed, hashOut)[0] != 0);
+    assertTrue(MurmurHash3v4.hash(s, seed, hashOut)[0] != 0);
     long v = 123;
-    assertTrue(MurmurHash3v3.hash(v, seed, hashOut)[0] != 0);
+    assertTrue(MurmurHash3v4.hash(v, seed, hashOut)[0] != 0);
   }
 
   @Test
@@ -415,8 +415,8 @@ private static long[] checkDouble(double dbl) {
 
     WritableMemory wmem = WritableMemory.writableWrap(dataArr);
     long[] hash1 = MurmurHash3.hash(dataArr, 0);
-    hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2);
-    long[] hash3 = MurmurHash3v3.hash(dbl, seed, hash2);
+    hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2);
+    long[] hash3 = MurmurHash3v4.hash(dbl, seed, hash2);
 
     assertEquals(hash1, hash2);
     assertEquals(hash1, hash3);

From ecfb4387b7164a1dc1da2495f902692d6589204e Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Sat, 8 Feb 2025 20:49:35 -0800
Subject: [PATCH 68/88] temporary change pom to 6.1.0-SNAPSHOT

---
 pom.xml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/pom.xml b/pom.xml
index c72af4f39..1d829adc4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,7 +33,7 @@ under the License.
 
   org.apache.datasketches
   datasketches-java
-  7.1.0-SNAPSHOT
+  8.0.0-SNAPSHOT
   jar
 
   ${project.artifactId}
@@ -83,7 +83,7 @@ under the License.
   
 
   
-    4.1.0
+    6.1.0-SNAPSHOT
 
     
     7.10.2
@@ -94,11 +94,11 @@ under the License.
 
     
     3.6.3
-    17
-    --add-modules=jdk.incubator.foreign
+    21
+    --enable-preview
     ${java.version}
     ${java.version}
-    -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 ${add-modules}
+    -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 ${jvm-ffm-flag}
     UTF-8
     ${charset.encoding}
     ${charset.encoding}
@@ -164,7 +164,7 @@ under the License.
           ${maven-compiler-plugin.version}
           
             
-              ${add-modules}
+              ${jvm-ffm-flag}
             
           
         
@@ -238,7 +238,7 @@ under the License.
             true
             public
             
-              ${add-modules}
+              ${jvm-ffm-flag}
             
           
           
@@ -284,7 +284,7 @@ under the License.
           maven-surefire-plugin
           ${maven-surefire-failsafe-plugins.version}
           
-            ${add-modules}
+            ${jvm-ffm-flag}
             false
             false
             true

From f33813370ed9c5c0a171a95190e0f7cfc0f6f56d Mon Sep 17 00:00:00 2001
From: AlexanderSaydakov 
Date: Wed, 12 Feb 2025 18:19:00 -0800
Subject: [PATCH 69/88] union(mem) delegates to union(sketch)

---
 .../apache/datasketches/theta/UnionImpl.java  | 106 +-----------------
 1 file changed, 2 insertions(+), 104 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/theta/UnionImpl.java b/src/main/java/org/apache/datasketches/theta/UnionImpl.java
index d5ae6071a..8c5b2f8f0 100644
--- a/src/main/java/org/apache/datasketches/theta/UnionImpl.java
+++ b/src/main/java/org/apache/datasketches/theta/UnionImpl.java
@@ -20,28 +20,17 @@
 package org.apache.datasketches.theta;
 
 import static java.lang.Math.min;
-import static org.apache.datasketches.theta.PreambleUtil.COMPACT_FLAG_MASK;
-import static org.apache.datasketches.theta.PreambleUtil.ORDERED_FLAG_MASK;
 import static org.apache.datasketches.theta.PreambleUtil.UNION_THETA_LONG;
 import static org.apache.datasketches.theta.PreambleUtil.clearEmpty;
-import static org.apache.datasketches.theta.PreambleUtil.extractCurCount;
 import static org.apache.datasketches.theta.PreambleUtil.extractFamilyID;
-import static org.apache.datasketches.theta.PreambleUtil.extractFlags;
-import static org.apache.datasketches.theta.PreambleUtil.extractLgArrLongs;
-import static org.apache.datasketches.theta.PreambleUtil.extractPreLongs;
-import static org.apache.datasketches.theta.PreambleUtil.extractSeedHash;
-import static org.apache.datasketches.theta.PreambleUtil.extractSerVer;
-import static org.apache.datasketches.theta.PreambleUtil.extractThetaLong;
 import static org.apache.datasketches.theta.PreambleUtil.extractUnionThetaLong;
 import static org.apache.datasketches.theta.PreambleUtil.insertUnionThetaLong;
-import static org.apache.datasketches.theta.SingleItemSketch.otherCheckForSingleItem;
 import static org.apache.datasketches.thetacommon.QuickSelect.selectExcludingZeros;
 
 import java.nio.ByteBuffer;
 
 import org.apache.datasketches.common.Family;
 import org.apache.datasketches.common.ResizeFactor;
-import org.apache.datasketches.common.SketchesArgumentException;
 import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.memory.MemoryRequestServer;
 import org.apache.datasketches.memory.WritableMemory;
@@ -347,99 +336,8 @@ public void union(final Sketch sketchIn) {
 
   @Override
   public void union(final Memory skMem) {
-    if (skMem == null) { return; }
-    final int cap = (int) skMem.getCapacity();
-    if (cap < 16) { return; } //empty or garbage
-    final int serVer = extractSerVer(skMem);
-    final int fam = extractFamilyID(skMem);
-
-    if (serVer == 4) { // compressed ordered compact
-      ThetaUtil.checkSeedHashes(expectedSeedHash_, (short) extractSeedHash(skMem));
-      union(CompactSketch.wrap(skMem));
-      return;
-    }
-    if (serVer == 3) { //The OpenSource sketches (Aug 4, 2015) starts with serVer = 3
-      if (fam < 1 || fam > 3) {
-        throw new SketchesArgumentException(
-            "Family must be Alpha, QuickSelect, or Compact: " + Family.idToFamily(fam));
-      }
-      processVer3(skMem);
-      return;
-    }
-    if (serVer == 2) { //older Sketch, which is compact and ordered
-      ThetaUtil.checkSeedHashes(expectedSeedHash_, (short)extractSeedHash(skMem));
-      union(ForwardCompatibility.heapify2to3(skMem, expectedSeedHash_));
-      return;
-    }
-    if (serVer == 1) { //much older Sketch, which is compact and ordered, no seedHash
-      union(ForwardCompatibility.heapify1to3(skMem, expectedSeedHash_));
-      return;
-    }
-    throw new SketchesArgumentException("SerVer is unknown: " + serVer);
-  }
-
-  //Has seedHash, p, could have 0 entries & theta < 1.0,
-  //could be unordered, ordered, compact, or not compact,
-  //could be Alpha, QuickSelect, or Compact.
-  private void processVer3(final Memory skMem) {
-    final int preLongs = extractPreLongs(skMem);
-
-    if (preLongs == 1) {
-      if (otherCheckForSingleItem(skMem)) {
-        final long hash = skMem.getLong(8);
-        gadget_.hashUpdate(hash);
-        return;
-      }
-      return; //empty
-    }
-    ThetaUtil.checkSeedHashes(expectedSeedHash_, (short)extractSeedHash(skMem));
-    final int curCountIn;
-    final long thetaLongIn;
-
-    if (preLongs == 2) { //exact mode
-      curCountIn = extractCurCount(skMem);
-      if (curCountIn == 0) { return; } //should be > 0, but if it is 0 return empty anyway.
-      thetaLongIn = Long.MAX_VALUE;
-    }
-
-    else { //prelongs == 3
-      //curCount may be 0 (e.g., from intersection); but sketch cannot be empty.
-      curCountIn = extractCurCount(skMem);
-      thetaLongIn = extractThetaLong(skMem);
-    }
-
-    unionThetaLong_ = min(min(unionThetaLong_, thetaLongIn), gadget_.getThetaLong()); //theta rule
-    unionEmpty_ = false;
-    final int flags = extractFlags(skMem);
-    final boolean ordered = (flags & ORDERED_FLAG_MASK) != 0;
-    if (ordered) { //must be compact
-
-      for (int i = 0; i < curCountIn; i++ ) {
-        final int offsetBytes = preLongs + i << 3;
-        final long hashIn = skMem.getLong(offsetBytes);
-        if (hashIn >= unionThetaLong_) { break; } // "early stop"
-        gadget_.hashUpdate(hashIn); //backdoor update, hash function is bypassed
-      }
-    }
-
-    else { //not-ordered, could be compact or hash-table form
-      final boolean compact = (flags & COMPACT_FLAG_MASK) != 0;
-      final int size = compact ? curCountIn : 1 << extractLgArrLongs(skMem);
-
-      for (int i = 0; i < size; i++ ) {
-        final int offsetBytes = preLongs + i << 3;
-        final long hashIn = skMem.getLong(offsetBytes);
-        if (hashIn <= 0L || hashIn >= unionThetaLong_) { continue; }
-        gadget_.hashUpdate(hashIn); //backdoor update, hash function is bypassed
-      }
-    }
-
-    unionThetaLong_ = min(unionThetaLong_, gadget_.getThetaLong()); //sync thetaLongs
-
-    if (gadget_.hasMemory()) {
-      final WritableMemory wmem = (WritableMemory)gadget_.getMemory();
-      PreambleUtil.insertUnionThetaLong(wmem, unionThetaLong_);
-      PreambleUtil.clearEmpty(wmem);
+    if (skMem != null) {
+      union(Sketch.wrap(skMem));
     }
   }
 

From 82b155fa907e786889a7697fea22512053c33214 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Mon, 17 Feb 2025 22:30:29 -0800
Subject: [PATCH 70/88] This is the first crack at updating ds-java to Java 21.

Lots of changes.
---
 .../datasketches/hash/MurmurHash3Adaptor.java | 479 ------------------
 .../datasketches/hash/MurmurHash3FFM21.java   | 386 ++++++++++++++
 .../datasketches/hll/DirectAuxHashMap.java    |   2 +-
 .../datasketches/kll/KllDoublesHelper.java    |   7 +-
 .../datasketches/kll/KllDoublesSketch.java    |   2 +-
 .../datasketches/kll/KllFloatsHelper.java     |   7 +-
 .../datasketches/kll/KllFloatsSketch.java     |   2 +-
 .../apache/datasketches/kll/KllHelper.java    |  14 +-
 .../datasketches/kll/KllLongsHelper.java      |   7 +-
 .../datasketches/kll/KllLongsSketch.java      |   2 +-
 .../quantiles/DirectUpdateDoublesSketch.java  |   3 +-
 .../quantiles/DirectUpdateDoublesSketchR.java |   2 +-
 .../datasketches/quantiles/DoublesSketch.java |   4 +-
 .../ConcurrentDirectQuickSelectSketch.java    |   4 +-
 .../ConcurrentHeapQuickSelectSketch.java      |   4 +-
 .../theta/DirectQuickSelectSketch.java        |   4 +-
 .../filters/bloomfilter/BloomFilterTest.java  |  26 +-
 .../hash/MurmurHash3AdaptorTest.java          | 377 --------------
 ...3v4Test.java => MurmurHash3FFM21Test.java} |  69 ++-
 .../hash/MurmurHash3FFM21bTest.java           | 419 +++++++++++++++
 .../hll/DirectAuxHashMapTest.java             |  11 +-
 .../hll/DirectCouponListTest.java             |   8 +-
 .../datasketches/kll/KllItemsSketchTest.java  |  13 +-
 .../quantiles/DebugUnionTest.java             |   7 +-
 .../DirectQuantilesMemoryRequestTest.java     |  32 +-
 .../quantiles/DoublesSketchTest.java          |  12 +-
 .../quantiles/PreambleUtilTest.java           |  11 +-
 .../datasketches/theta/CompactSketchTest.java |   6 +-
 .../theta/DirectQuickSelectSketchTest.java    | 104 ++--
 .../theta/HeapifyWrapSerVer1and2Test.java     | 106 ++--
 .../datasketches/theta/SetOperationTest.java  |  10 +-
 .../datasketches/theta/UnionImplTest.java     |  19 +-
 32 files changed, 1068 insertions(+), 1091 deletions(-)
 delete mode 100644 src/main/java/org/apache/datasketches/hash/MurmurHash3Adaptor.java
 create mode 100644 src/main/java/org/apache/datasketches/hash/MurmurHash3FFM21.java
 delete mode 100644 src/test/java/org/apache/datasketches/hash/MurmurHash3AdaptorTest.java
 rename src/test/java/org/apache/datasketches/hash/{MurmurHash3v4Test.java => MurmurHash3FFM21Test.java} (83%)
 create mode 100644 src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java

diff --git a/src/main/java/org/apache/datasketches/hash/MurmurHash3Adaptor.java b/src/main/java/org/apache/datasketches/hash/MurmurHash3Adaptor.java
deleted file mode 100644
index feecdba97..000000000
--- a/src/main/java/org/apache/datasketches/hash/MurmurHash3Adaptor.java
+++ /dev/null
@@ -1,479 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.datasketches.hash;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.datasketches.common.Util.ceilingPowerOf2;
-import static org.apache.datasketches.hash.MurmurHash3.hash;
-
-import java.nio.ByteBuffer;
-
-import org.apache.datasketches.common.SketchesArgumentException;
-import org.apache.datasketches.common.SketchesStateException;
-
-/**
- * A general purpose wrapper for the MurmurHash3.
- * 
    
- * 
  • Inputs can be long, long[], int[], char[], byte[], double or String.
    • 
- * 
  • Returns null if arrays or String is null or empty.
    • 
- * 
  • Provides methods for returning the 128-bit result as either an array of 2 longs or as a byte
- * array of 16 bytes.
    • 
- * 
  • Provides modulo, asDouble and asInt functions.
    • 
- * 

- *
- * @author Lee Rhodes
- */
-public final class MurmurHash3Adaptor {
-  private static final long BIT62 = 1L << 62;
-  private static final long MAX_LONG = Long.MAX_VALUE;
-  private static final long INT_MASK = 0x7FFFFFFFL;
-  private static final long PRIME = 9219741426499971445L; //from P. L'Ecuyer and R. Simard
-
-  private MurmurHash3Adaptor() {}
-
-  /**
-   * Hash a long and long seed.
-   *
-   * @param datum the input long value
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a byte[16] in Big Endian order from 2 64-bit longs.
-   */
-  public static byte[] hashToBytes(final long datum, final long seed) {
-    final long[] data = { datum };
-    return toByteArray(hash(data, seed));
-  }
-
-  /**
-   * Hash a long[] and long seed.
-   *
-   * @param data the input long array
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a byte[16] in Big Endian order from 2 64-bit longs.
-   */
-  public static byte[] hashToBytes(final long[] data, final long seed) {
-    if ((data == null) || (data.length == 0)) {
-      return null;
-    }
-    return toByteArray(hash(data, seed));
-  }
-
-  /**
-   * Hash an int[] and long seed.
-   *
-   * @param data the input int array
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a byte[16] in Big Endian order from 2 64-bit longs.
-   */
-  public static byte[] hashToBytes(final int[] data, final long seed) {
-    if ((data == null) || (data.length == 0)) {
-      return null;
-    }
-    return toByteArray(hash(data, seed));
-  }
-
-  /**
-   * Hash a char[] and long seed.
-   *
-   * @param data the input char array
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a byte[16] in Big Endian order from 2 64-bit longs.
-   */
-  public static byte[] hashToBytes(final char[] data, final long seed) {
-    if ((data == null) || (data.length == 0)) {
-      return null;
-    }
-    return toByteArray(hash(data, seed));
-  }
-
-  /**
-   * Hash a byte[] and long seed.
-   *
-   * @param data the input byte array
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a byte[16] in Big Endian order from 2 64-bit longs.
-   */
-  public static byte[] hashToBytes(final byte[] data, final long seed) {
-    if ((data == null) || (data.length == 0)) {
-      return null;
-    }
-    return toByteArray(hash(data, seed));
-  }
-
-  /**
-   * Hash a double and long seed.
-   *
-   * @param datum the input double
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a byte[16] in Big Endian order from 2 64-bit longs.
-   */
-  public static byte[] hashToBytes(final double datum, final long seed) {
-    final double d = (datum == 0.0) ? 0.0 : datum; //canonicalize -0.0, 0.0
-    final long[] data = { Double.doubleToLongBits(d) }; //canonicalize all NaN forms
-    return toByteArray(hash(data, seed));
-  }
-
-  /**
-   * Hash a String and long seed.
-   *
-   * @param datum the input String
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a byte[16] in Big Endian order from 2 64-bit longs.
-   */
-  public static byte[] hashToBytes(final String datum, final long seed) {
-    if ((datum == null) || datum.isEmpty()) {
-      return null;
-    }
-    final byte[] data = datum.getBytes(UTF_8);
-    return toByteArray(hash(data, seed));
-  }
-
-  /**
-   * Hash a long and long seed.
-   *
-   * @param datum the input long
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a long[2].
-   */
-  public static long[] hashToLongs(final long datum, final long seed) {
-    final long[] data = { datum };
-    return hash(data, seed);
-  }
-
-  /**
-   * Hash a long[] and long seed.
-   *
-   * @param data the input long array.
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a long[2].
-   */
-  public static long[] hashToLongs(final long[] data, final long seed) {
-    if ((data == null) || (data.length == 0)) {
-      return null;
-    }
-    return hash(data, seed);
-  }
-
-  /**
-   * Hash a int[] and long seed.
-   *
-   * @param data the input int array.
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a long[2].
-   */
-  public static long[] hashToLongs(final int[] data, final long seed) {
-    if ((data == null) || (data.length == 0)) {
-      return null;
-    }
-    return hash(data, seed);
-  }
-
-  /**
-   * Hash a char[] and long seed.
-   *
-   * @param data the input char array.
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a long[2].
-   */
-  public static long[] hashToLongs(final char[] data, final long seed) {
-    if ((data == null) || (data.length == 0)) {
-      return null;
-    }
-    return hash(data, seed);
-  }
-
-  /**
-   * Hash a byte[] and long seed.
-   *
-   * @param data the input byte array.
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a long[2].
-   */
-  public static long[] hashToLongs(final byte[] data, final long seed) {
-    if ((data == null) || (data.length == 0)) {
-      return null;
-    }
-    return hash(data, seed);
-  }
-
-  /**
-   * Hash a double and long seed.
-   *
-   * @param datum the input double.
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a long[2].
-   */
-  public static long[] hashToLongs(final double datum, final long seed) {
-    final double d = (datum == 0.0) ? 0.0 : datum; //canonicalize -0.0, 0.0
-    final long[] data = { Double.doubleToLongBits(d) };//canonicalize all NaN forms
-    return hash(data, seed);
-  }
-
-  /**
-   * Hash a String and long seed.
-   *
-   * @param datum the input String.
-   * @param seed A long valued seed.
-   * @return The 128-bit hash as a long[2].
-   */
-  public static long[] hashToLongs(final String datum, final long seed) {
-    if ((datum == null) || datum.isEmpty()) {
-      return null;
-    }
-    final byte[] data = datum.getBytes(UTF_8);
-    return hash(data, seed);
-  }
-
-  //As Integer functions
-
-  /**
-   * Returns a deterministic uniform random integer between zero (inclusive) and
-   * n (exclusive) given the input data.
-   * @param data the input long array.
-   * @param n The upper exclusive bound of the integers produced. Must be > 1.
-   * @return deterministic uniform random integer
-   */
-  public static int asInt(final long[] data, final int n) {
-    if ((data == null) || (data.length == 0)) {
-      throw new SketchesArgumentException("Input is null or empty.");
-    }
-    return asInteger(data, n); //data is long[]
-  }
-
-  /**
-   * Returns a deterministic uniform random integer between zero (inclusive) and
-   * n (exclusive) given the input data.
-   * @param data the input int array.
-   * @param n The upper exclusive bound of the integers produced. Must be > 1.
-   * @return deterministic uniform random integer
-   */
-  public static int asInt(final int[] data, final int n) {
-    if ((data == null) || (data.length == 0)) {
-      throw new SketchesArgumentException("Input is null or empty.");
-    }
-    return asInteger(toLongArray(data), n); //data is int[]
-  }
-
-  /**
-   * Returns a deterministic uniform random integer between zero (inclusive) and
-   * n (exclusive) given the input data.
-   * @param data the input byte array.
-   * @param n The upper exclusive bound of the integers produced. Must be > 1.
-   * @return deterministic uniform random integer.
-   */
-  public static int asInt(final byte[] data, final int n) {
-    if ((data == null) || (data.length == 0)) {
-      throw new SketchesArgumentException("Input is null or empty.");
-    }
-    return asInteger(toLongArray(data), n); //data is byte[]
-  }
-
-  /**
-   * Returns a deterministic uniform random integer between zero (inclusive) and
-   * n (exclusive) given the input datum.
-   * @param datum the input long
-   * @param n The upper exclusive bound of the integers produced. Must be > 1.
-   * @return deterministic uniform random integer
-   */
-  public static int asInt(final long datum, final int n) {
-    final long[] data = { datum };
-    return asInteger(data, n); //data is long[]
-  }
-
-  /**
-   * Returns a deterministic uniform random integer between zero (inclusive) and
-   * n (exclusive) given the input double.
-   * @param datum the given double.
-   * @param n The upper exclusive bound of the integers produced. Must be > 1.
-   * @return deterministic uniform random integer
-   */
-  public static int asInt(final double datum, final int n) {
-    final double d = (datum == 0.0) ? 0.0 : datum; //canonicalize -0.0, 0.0
-    final long[] data = { Double.doubleToLongBits(d) };//canonicalize all NaN forms
-    return asInteger(data, n); //data is long[]
-  }
-
-  /**
-   * Returns a deterministic uniform random integer between zero (inclusive) and
-   * n (exclusive) given the input datum.
-   * @param datum the given String.
-   * @param n The upper exclusive bound of the integers produced. Must be > 1.
-   * @return deterministic uniform random integer
-   */
-  public static int asInt(final String datum, final int n) {
-    if ((datum == null) || datum.isEmpty()) {
-      throw new SketchesArgumentException("Input is null or empty.");
-    }
-    final byte[] data = datum.getBytes(UTF_8);
-    return asInteger(toLongArray(data), n); //data is byte[]
-  }
-
-  /**
-   * Returns a deterministic uniform random integer with a minimum inclusive value of zero and a
-   * maximum exclusive value of n given the input data.
-   *
-   * 
The integer values produced are only as random as the MurmurHash3 algorithm, which may be
-   * adequate for many applications. However, if you are looking for high guarantees of randomness
-   * you should turn to more sophisticated random generators such as Mersenne Twister or Well19937c
-   * algorithms.
-   *
-   * @param data The input data (key)
-   * @param n The upper exclusive bound of the integers produced. Must be > 1.
-   * @return deterministic uniform random integer
-   */
-  private static int asInteger(final long[] data, final int n) {
-    int t;
-    int cnt = 0;
-    long seed = 0;
-    if (n < 2) {
-      throw new SketchesArgumentException("Given value of n must be > 1.");
-    }
-    if (n > (1 << 30)) {
-      while (++cnt < 10000) {
-        final long[] h = MurmurHash3.hash(data, seed);
-        t = (int) (h[0] & INT_MASK);
-        if (t < n) {
-          return t;
-        }
-        t = (int) ((h[0] >>> 33));
-        if (t < n) {
-          return t;
-        }
-        t = (int) (h[1] & INT_MASK);
-        if (t < n) {
-          return t;
-        }
-        t = (int) ((h[1] >>> 33));
-        if (t < n) {
-          return t;
-        }
-        seed += PRIME;
-      } // end while
-      throw new SketchesStateException(
-          "Internal Error: Failed to find integer < n within 10000 iterations.");
-    }
-    final long mask = ceilingPowerOf2(n) - 1;
-    while (++cnt < 10000) {
-      final long[] h = MurmurHash3.hash(data, seed);
-      t = (int) (h[0] & mask);
-      if (t < n) {
-        return t;
-      }
-      t = (int) ((h[0] >>> 33) & mask);
-      if (t < n) {
-        return t;
-      }
-      t = (int) (h[1] & mask);
-      if (t < n) {
-        return t;
-      }
-      t = (int) ((h[1] >>> 33) & mask);
-      if (t < n) {
-        return t;
-      }
-      seed += PRIME;
-    } // end while
-    throw new SketchesStateException(
-        "Internal Error: Failed to find integer < n within 10000 iterations.");
-  }
-
-  /**
-   * Returns a uniform random double with a minimum inclusive value of zero and a maximum exclusive
-   * value of 1.0.
-   *
-   * 
The double values produced are only as random as the MurmurHash3 algorithm, which may be
-   * adequate for many applications. However, if you are looking for high guarantees of randomness
-   * you should turn to more sophisticated random generators such as Mersenne Twister or Well
-   * algorithms.
-   *
-   * @param hash The output of the MurmurHash3.
-   * @return the uniform random double.
-   */
-  public static double asDouble(final long[] hash) {
-    return (hash[0] >>> 12) * 0x1.0p-52d;
-  }
-
-  /**
-   * Returns the remainder from the modulo division of the 128-bit output of the murmurHash3 by the
-   * divisor.
-   *
-   * @param h0 The lower 64-bits of the 128-bit MurmurHash3 hash.
-   * @param h1 The upper 64-bits of the 128-bit MurmurHash3 hash.
-   * @param divisor Must be positive and greater than zero.
-   * @return the modulo result.
-   */
-  public static int modulo(final long h0, final long h1, final int divisor) {
-    final long d = divisor;
-    final long modH0 = (h0 < 0L) ? addRule(mulRule(BIT62, 2L, d), (h0 & MAX_LONG), d) : h0 % d;
-    final long modH1 = (h1 < 0L) ? addRule(mulRule(BIT62, 2L, d), (h1 & MAX_LONG), d) : h1 % d;
-    final long modTop = mulRule(mulRule(BIT62, 4L, d), modH1, d);
-    return (int) addRule(modTop, modH0, d);
-  }
-
-  /**
-   * Returns the remainder from the modulo division of the 128-bit output of the murmurHash3 by the
-   * divisor.
-   *
-   * @param hash The size 2 long array from the MurmurHash3.
-   * @param divisor Must be positive and greater than zero.
-   * @return the modulo result
-   */
-  public static int modulo(final long[] hash, final int divisor) {
-    return modulo(hash[0], hash[1], divisor);
-  }
-
-  private static long addRule(final long a, final long b, final long d) {
-    return ((a % d) + (b % d)) % d;
-  }
-
-  private static long mulRule(final long a, final long b, final long d) {
-    return ((a % d) * (b % d)) % d;
-  }
-
-  private static byte[] toByteArray(final long[] hash) { //Assumes Big Endian
-    final byte[] bArr = new byte[16];
-    final ByteBuffer bb = ByteBuffer.wrap(bArr);
-    bb.putLong(hash[0]);
-    bb.putLong(hash[1]);
-    return bArr;
-  }
-
-  private static long[] toLongArray(final byte[] data) {
-    final int dataLen = data.length;
-    final int longLen = (dataLen + 7) / 8;
-    final long[] longArr = new long[longLen];
-    for (int bi = 0; bi < dataLen; bi++) {
-      final int li = bi / 8;
-      longArr[li] |= (((long)data[bi]) << ((bi * 8) % 64));
-    }
-    return longArr;
-  }
-
-  private static long[] toLongArray(final int[] data) {
-    final int dataLen = data.length;
-    final int longLen = (dataLen + 1) / 2;
-    final long[] longArr = new long[longLen];
-    for (int ii = 0; ii < dataLen; ii++) {
-      final int li = ii / 2;
-      longArr[li] |= (((long)data[ii]) << ((ii * 32) % 64));
-    }
-    return longArr;
-  }
-
-}
diff --git a/src/main/java/org/apache/datasketches/hash/MurmurHash3FFM21.java b/src/main/java/org/apache/datasketches/hash/MurmurHash3FFM21.java
new file mode 100644
index 000000000..a94fd3fd9
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hash/MurmurHash3FFM21.java
@@ -0,0 +1,386 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hash;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.ValueLayout;
+import java.util.Objects;
+
+import org.apache.datasketches.memory.Memory;
+
+/**
+ * The MurmurHash3 is a fast, non-cryptographic, 128-bit hash function that has
+ * excellent avalanche and 2-way bit independence properties.
+ *
+ * 
Austin Appleby's C++
+ * 
+ * MurmurHash3_x64_128(...), final revision 150,
+ * which is in the Public Domain, was the inspiration for this implementation in Java.

+ *
+ * 
This implementation of the MurmurHash3 allows hashing of a block of on-heap Memory defined by an offset
+ * and length. The calling API also allows the user to supply the small output array of two longs,
+ * so that the entire hash function is static and free of object allocations.

+ *
+ * 
This implementation produces exactly the same hash result as the
+ * MurmurHash3 function in datasketches-java given compatible inputs.

+ *
+ * 
This FFM21 version of the implementation leverages the java.lang.foreign package (FFM) of JDK-21 in place of
+ * the Unsafe class.
+ *
+ * @author Lee Rhodes
+ */
+public final class MurmurHash3FFM21 {
+  private static final long C1 = 0x87c37b91114253d5L;
+  private static final long C2 = 0x4cf5ad432745937fL;
+
+  /**
+   * Returns a 128-bit hash of the input.
+   * Provided for compatibility with older version of MurmurHash3,
+   * but empty or null input now throws IllegalArgumentException.
+   * @param in long array
+   * @param seed A long valued seed.
+   * @return the hash
+   * @throws IllegalArgumentException if input is empty or null
+   */
+  public static long[] hash(final long[] in, final long seed) {
+    if ((in == null) || (in.length == 0)) {
+      throw new IllegalArgumentException("Input in is empty or null.");
+    }
+    return hash(MemorySegment.ofArray(in), 0L, in.length << 3, seed, new long[2]);
+  }
+
+  /**
+   * Returns a 128-bit hash of the input.
+   * Provided for compatibility with older version of MurmurHash3,
+   * but empty or null input now throws IllegalArgumentException.
+   * @param in int array
+   * @param seed A long valued seed.
+   * @return the hash
+   * @throws IllegalArgumentException if input is empty or null
+   */
+  public static long[] hash(final int[] in, final long seed) {
+    if ((in == null) || (in.length == 0)) {
+      throw new IllegalArgumentException("Input in is empty or null.");
+    }
+    return hash(MemorySegment.ofArray(in), 0L, in.length << 2, seed, new long[2]);
+  }
+
+  /**
+   * Returns a 128-bit hash of the input.
+   * Provided for compatibility with older version of MurmurHash3,
+   * but empty or null input now throws IllegalArgumentException.
+   * @param in char array
+   * @param seed A long valued seed.
+   * @return the hash
+   * @throws IllegalArgumentException if input is empty or null
+   */
+  public static long[] hash(final char[] in, final long seed) {
+    if ((in == null) || (in.length == 0)) {
+      throw new IllegalArgumentException("Input in is empty or null.");
+    }
+    return hash(MemorySegment.ofArray(in), 0L, in.length << 1, seed, new long[2]);
+  }
+
+  /**
+   * Returns a 128-bit hash of the input.
+   * Provided for compatibility with older version of MurmurHash3,
+   * but empty or null input now throws IllegalArgumentException.
+   * @param in byte array
+   * @param seed A long valued seed.
+   * @return the hash
+   * @throws IllegalArgumentException if input is empty or null
+   */
+  public static long[] hash(final byte[] in, final long seed) {
+    if ((in == null) || (in.length == 0)) {
+      throw new IllegalArgumentException("Input in is empty or null.");
+    }
+    return hash(MemorySegment.ofArray(in), 0L, in.length, seed, new long[2]);
+  }
+
+  //Single primitive inputs
+
+  /**
+   * Returns a 128-bit hash of the input.
+   * Note the entropy of the resulting hash cannot be more than 64 bits.
+   * @param in a long
+   * @param seed A long valued seed.
+   * @param hashOut A long array of size 2
+   * @return the hash
+   */
+  public static long[] hash(final long in, final long seed, final long[] hashOut) {
+    final long h1 = seed ^ mixK1(in);
+    final long h2 = seed;
+    return finalMix128(h1, h2, 8, hashOut);
+  }
+
+  /**
+   * Returns a 128-bit hash of the input.
+   * Note the entropy of the resulting hash cannot be more than 64 bits.
+   * @param in a double
+   * @param seed A long valued seed.
+   * @param hashOut A long array of size 2
+   * @return the hash
+   */
+  public static long[] hash(final double in, final long seed, final long[] hashOut) {
+    final double d = (in == 0.0) ? 0.0 : in;    // canonicalize -0.0, 0.0
+    final long k1 = Double.doubleToLongBits(d); // canonicalize all NaN forms
+    final long h1 = seed ^ mixK1(k1);
+    final long h2 = seed;
+    return finalMix128(h1, h2, 8, hashOut);
+  }
+
+  /**
+   * Returns a 128-bit hash of the input.
+   * An empty or null input throws IllegalArgumentException.
+   * @param in a String
+   * @param seed A long valued seed.
+   * @param hashOut A long array of size 2
+   * @return the hash
+   * @throws IllegalArgumentException if input is empty or null
+   */
+  public static long[] hash(final String in, final long seed, final long[] hashOut) {
+    if ((in == null) || (in.length() == 0)) {
+      throw new IllegalArgumentException("Input in is empty or null.");
+    }
+    final byte[] byteArr = in.getBytes(UTF_8);
+    return hash(MemorySegment.ofArray(byteArr), 0L, byteArr.length, seed, hashOut);
+  }
+
+  //The main API calls
+
+  /**
+   * Returns a 128-bit hash of the input as a long array of size 2.
+   *
+   * @param mem The input Memory. Must be non-null and non-empty,
+   * otherwise throws IllegalArgumentException.
+   * @param offsetBytes the starting point within Memory.
+   * @param lengthBytes the total number of bytes to be hashed.
+   * @param seed A long valued seed.
+   * @param hashOut the size 2 long array for the resulting 128-bit hash
+   * @return the hash.
+   */
+  public static long[] hash(final Memory mem, final long offsetBytes, final long lengthBytes,
+      final long seed, final long[] hashOut) {
+    Objects.requireNonNull(mem, "Input Memory is null");
+    final MemorySegment seg = mem.getMemorySegment();
+    return hash(seg, offsetBytes, lengthBytes, seed, hashOut);
+  }
+
+  /**
+   * Returns a 128-bit hash of the input as a long array of size 2.
+   *
+   * @param seg The input MemorySegment. Must be non-null and non-empty,
+   * otherwise throws IllegalArgumentException.
+   * @param offsetBytes the starting point within Memory.
+   * @param lengthBytes the total number of bytes to be hashed.
+   * @param seed A long valued seed.
+   * @param hashOut the size 2 long array for the resulting 128-bit hash
+   * @return the hash.
+   * @throws IllegalArgumentException if input MemorySegment is empty
+   */
+  public static long[] hash(final MemorySegment seg, final long offsetBytes, final long lengthBytes,
+      final long seed, final long[] hashOut) {
+    Objects.requireNonNull(seg, "Input MemorySegment is null");
+    if (seg.byteSize() == 0L) { throw new IllegalArgumentException("Input MemorySegment is empty."); }
+
+    long cumOff = offsetBytes;
+
+    long h1 = seed;
+    long h2 = seed;
+    long rem = lengthBytes;
+
+    // Process the 128-bit blocks (the body) into the hash
+    while (rem >= 16L) {
+      final long k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff);     //0, 16, 32, ...
+      final long k2 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff + 8); //8, 24, 40, ...
+      cumOff += 16L;
+      rem -= 16L;
+
+      h1 ^= mixK1(k1);
+      h1 = Long.rotateLeft(h1, 27);
+      h1 += h2;
+      h1 = (h1 * 5) + 0x52dce729L;
+
+      h2 ^= mixK2(k2);
+      h2 = Long.rotateLeft(h2, 31);
+      h2 += h1;
+      h2 = (h2 * 5) + 0x38495ab5L;
+    }
+
+    // Get the tail (if any): 1 to 15 bytes
+    if (rem > 0L) {
+      long k1 = 0;
+      long k2 = 0;
+      switch ((int) rem) {
+        case 15: {
+          k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 14) & 0xFFL) << 48;
+        }
+        //$FALL-THROUGH$
+        case 14: {
+          k2 ^= (seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 12) & 0xFFFFL) << 32;
+          k2 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL;
+          k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff);
+          break;
+        }
+
+        case 13: {
+          k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 12) & 0xFFL) << 32;
+        }
+        //$FALL-THROUGH$
+        case 12: {
+          k2 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL;
+          k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff);
+          break;
+        }
+
+        case 11: {
+          k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 10) & 0xFFL) << 16;
+        }
+        //$FALL-THROUGH$
+        case 10: {
+          k2 ^= seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 8) & 0xFFFFL;
+          k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff);
+          break;
+        }
+
+        case  9: {
+          k2 ^= seg.get(ValueLayout.JAVA_BYTE, cumOff + 8) & 0xFFL;
+        }
+        //$FALL-THROUGH$
+        case  8: {
+          k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff);
+          break;
+        }
+
+        case  7: {
+          k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 6) & 0xFFL) << 48;
+        }
+        //$FALL-THROUGH$
+        case  6: {
+          k1 ^= (seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 4) & 0xFFFFL) << 32;
+          k1 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff) & 0xFFFFFFFFL;
+          break;
+        }
+
+        case  5: {
+          k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 4) & 0xFFL) << 32;
+        }
+        //$FALL-THROUGH$
+        case  4: {
+          k1 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff) & 0xFFFFFFFFL;
+          break;
+        }
+
+        case  3: {
+          k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 2) & 0xFFL) << 16;
+        }
+        //$FALL-THROUGH$
+        case  2: {
+          k1 ^= seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff) & 0xFFFFL;
+          break;
+        }
+
+        case  1: {
+          k1 ^= seg.get(ValueLayout.JAVA_BYTE, cumOff) & 0xFFL;
+          break;
+        }
+        default: break; //can't happen
+      }
+
+      h1 ^= mixK1(k1);
+      h2 ^= mixK2(k2);
+    }
+    return finalMix128(h1, h2, lengthBytes, hashOut);
+  }
+
+  //--Helper methods----------------------------------------------------
+
+  /**
+   * Self mix of k1
+   *
+   * @param k1 input argument
+   * @return mix
+   */
+  private static long mixK1(long k1) {
+    k1 *= C1;
+    k1 = Long.rotateLeft(k1, 31);
+    k1 *= C2;
+    return k1;
+  }
+
+  /**
+   * Self mix of k2
+   *
+   * @param k2 input argument
+   * @return mix
+   */
+  private static long mixK2(long k2) {
+    k2 *= C2;
+    k2 = Long.rotateLeft(k2, 33);
+    k2 *= C1;
+    return k2;
+  }
+
+  /**
+   * Final self mix of h*.
+   *
+   * @param h input to final mix
+   * @return mix
+   */
+  private static long finalMix64(long h) {
+    h ^= h >>> 33;
+    h *= 0xff51afd7ed558ccdL;
+    h ^= h >>> 33;
+    h *= 0xc4ceb9fe1a85ec53L;
+    h ^= h >>> 33;
+    return h;
+  }
+
+  /**
+   * Finalization: Add the length into the hash and mix
+   * @param h1 intermediate hash
+   * @param h2 intermediate hash
+   * @param lengthBytes the length in bytes
+   * @param hashOut the output array of 2 longs
+   * @return hashOut
+   */
+  private static long[] finalMix128(long h1, long h2, final long lengthBytes, final long[] hashOut) {
+    h1 ^= lengthBytes;
+    h2 ^= lengthBytes;
+
+    h1 += h2;
+    h2 += h1;
+
+    h1 = finalMix64(h1);
+    h2 = finalMix64(h2);
+
+    h1 += h2;
+    h2 += h1;
+
+    hashOut[0] = h1;
+    hashOut[1] = h2;
+    return hashOut;
+  }
+
+  private MurmurHash3FFM21() { }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java b/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java
index 98884f5ea..e6c408f35 100644
--- a/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java
+++ b/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java
@@ -195,7 +195,7 @@ private static final void grow(final DirectHllArray host, final int oldLgAuxArrI
       final WritableMemory newWmem = svr.request(host.wmem, requestBytes);
       host.wmem.copyTo(0, newWmem, 0, host.auxStart);
       newWmem.clear(host.auxStart, newAuxBytes); //clear space for new aux data
-      svr.requestClose(host.wmem, newWmem); //old host.wmem is now invalid
+      svr.requestClose(host.wmem); //old host.wmem is now invalid
       host.updateMemory(newWmem);
     }
     //rehash into larger aux array
diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java
index acbecdf07..cb5788a75 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java
@@ -207,8 +207,11 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllDoublesSke
 
       //MEMORY SPACE MANAGEMENT
       if (mySketch.getWritableMemory() != null) {
-        final WritableMemory wmem =
-            KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewDoubleItemsArr.length);
+        final WritableMemory oldWmem = mySketch.getWritableMemory();
+        final WritableMemory wmem = KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewDoubleItemsArr.length);
+        if (!wmem.isSameResource(oldWmem)) {
+          mySketch.getMemoryRequestServer().requestClose(oldWmem);
+        }
         mySketch.setWritableMemory(wmem);
       }
     } //end of updating levels above level 0
diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
index fbe9dbb36..01608ca02 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
@@ -137,7 +137,7 @@ public static KllDoublesSketch wrap(final Memory srcMem) {
     Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null");
     final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, DOUBLES_SKETCH, null);
     if (memVal.sketchStructure == UPDATABLE) {
-      final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); //dummy
+      final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer();
       return new KllDirectDoublesSketch(memVal.sketchStructure, (WritableMemory)srcMem, memReqSvr, memVal);
     } else {
       return new KllDirectCompactDoublesSketch(memVal.sketchStructure, srcMem, memVal);
diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java
index 69045f78c..1796cac4d 100644
--- a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java
@@ -207,8 +207,11 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllFloatsSketch
 
       //MEMORY SPACE MANAGEMENT
       if (mySketch.getWritableMemory() != null) {
-        final WritableMemory wmem =
-            KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewFloatItemsArr.length);
+        final WritableMemory oldWmem = mySketch.getWritableMemory();
+        final WritableMemory wmem = KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewFloatItemsArr.length);
+        if (!wmem.isSameResource(oldWmem)) {
+          mySketch.getMemoryRequestServer().requestClose(oldWmem);
+        }
         mySketch.setWritableMemory(wmem);
       }
     } //end of updating levels above level 0
diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
index b993f9998..cf282c872 100644
--- a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
@@ -137,7 +137,7 @@ public static KllFloatsSketch wrap(final Memory srcMem) {
     Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null");
     final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, FLOATS_SKETCH, null);
     if (memVal.sketchStructure == UPDATABLE) {
-      final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); //dummy
+      final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer();
       return new KllDirectFloatsSketch(memVal.sketchStructure, (WritableMemory)srcMem, memReqSvr, memVal);
     } else {
       return new KllDirectCompactFloatsSketch(memVal.sketchStructure, srcMem, memVal);
diff --git a/src/main/java/org/apache/datasketches/kll/KllHelper.java b/src/main/java/org/apache/datasketches/kll/KllHelper.java
index 21188255c..8dd729da7 100644
--- a/src/main/java/org/apache/datasketches/kll/KllHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllHelper.java
@@ -52,6 +52,7 @@
 import org.apache.datasketches.common.SketchesArgumentException;
 import org.apache.datasketches.kll.KllSketch.SketchStructure;
 import org.apache.datasketches.kll.KllSketch.SketchType;
+import org.apache.datasketches.memory.MemoryRequestServer;
 import org.apache.datasketches.memory.WritableBuffer;
 import org.apache.datasketches.memory.WritableMemory;
 
@@ -353,7 +354,8 @@ static WritableMemory memorySpaceMgmt(
     final WritableMemory newWmem;
 
     if (requiredSketchBytes > oldWmem.getCapacity()) { //Acquire new WritableMemory
-      newWmem = sketch.getMemoryRequestServer().request(oldWmem, requiredSketchBytes);
+      final MemoryRequestServer memReqSvr = sketch.getMemoryRequestServer();
+      newWmem = memReqSvr.request(oldWmem, requiredSketchBytes);
       oldWmem.copyTo(0, newWmem, 0, DATA_START_ADR); //copy preamble (first 20 bytes)
     }
     else { //Expand or contract in current memory
@@ -617,7 +619,7 @@ else if (sketchType == FLOATS_SKETCH) {
       maxFloat = fltSk.getMaxItem();
       //assert we are following a certain growth scheme
       assert myCurFloatItemsArr.length == myCurTotalItemsCapacity;
-    } 
+    }
     else if (sketchType == LONGS_SKETCH) {
       final KllLongsSketch lngSk = (KllLongsSketch) sketch;
       myCurLongItemsArr = lngSk.getLongItemsArray();
@@ -668,7 +670,7 @@ else if (sketchType == FLOATS_SKETCH) {
       myNewFloatItemsArr = new float[myNewTotalItemsCapacity];
       // copy and shift the current items data into the new array
       System.arraycopy(myCurFloatItemsArr, 0, myNewFloatItemsArr, deltaItemsCap, myCurTotalItemsCapacity);
-    } 
+    }
     else if (sketchType == LONGS_SKETCH) {
       myNewLongItemsArr = new long[myNewTotalItemsCapacity];
       // copy and shift the current items data into the new array
@@ -682,7 +684,11 @@ else if (sketchType == LONGS_SKETCH) {
 
     //MEMORY SPACE MANAGEMENT
     if (sketch.getWritableMemory() != null) {
+      final WritableMemory oldWmem = sketch.getWritableMemory();
       final WritableMemory wmem = memorySpaceMgmt(sketch, myNewLevelsArr.length, myNewTotalItemsCapacity);
+      if (!wmem.isSameResource(oldWmem)) {
+        sketch.getMemoryRequestServer().requestClose(oldWmem);
+      }
       sketch.setWritableMemory(wmem);
     }
 
@@ -700,7 +706,7 @@ else if (sketchType == FLOATS_SKETCH) {
       fltSk.setMinItem(minFloat);
       fltSk.setMaxItem(maxFloat);
       fltSk.setFloatItemsArray(myNewFloatItemsArr);
-    } 
+    }
     else if (sketchType == LONGS_SKETCH) {
       final KllLongsSketch lngSk = (KllLongsSketch) sketch;
       lngSk.setMinItem(minLong);
diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java b/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java
index 04fe2cc08..2dc146103 100644
--- a/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java
@@ -207,8 +207,11 @@ static void mergeLongsImpl(final KllLongsSketch mySketch, final KllLongsSketch o
 
       //MEMORY SPACE MANAGEMENT
       if (mySketch.getWritableMemory() != null) {
-        final WritableMemory wmem =
-            KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewLongItemsArray.length);
+        final WritableMemory oldWmem = mySketch.getWritableMemory();
+        final WritableMemory wmem = KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewLongItemsArray.length);
+        if (!wmem.isSameResource(oldWmem)) {
+          mySketch.getMemoryRequestServer().requestClose(oldWmem);
+        }
         mySketch.setWritableMemory(wmem);
       }
     } //end of updating levels above level 0
diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java
index f5688ad70..827f825f8 100644
--- a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java
@@ -137,7 +137,7 @@ public static KllLongsSketch wrap(final Memory srcMem) {
     Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null");
     final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, LONGS_SKETCH, null);
     if (memVal.sketchStructure == UPDATABLE) {
-      final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); //dummy
+      final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer();
       return new KllDirectLongsSketch(memVal.sketchStructure, (WritableMemory)srcMem, memReqSvr, memVal);
     } else {
       return new KllDirectCompactLongsSketch(memVal.sketchStructure, srcMem, memVal);
diff --git a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java
index 7a7a7a850..12604ecb3 100644
--- a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java
@@ -266,9 +266,8 @@ private WritableMemory growCombinedMemBuffer(final int itemSpaceNeeded) {
     }
 
     final WritableMemory newMem = memReqSvr.request(mem_, needBytes);
-
     mem_.copyTo(0, newMem, 0, memBytes);
-    memReqSvr.requestClose(mem_, newMem);
+    memReqSvr.requestClose(mem_);
 
     return newMem;
   }
diff --git a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketchR.java b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketchR.java
index 3e80b3f0f..234170331 100644
--- a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketchR.java
+++ b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketchR.java
@@ -146,7 +146,7 @@ int getBaseBufferCount() {
 
   @Override
   int getCombinedBufferItemCapacity() {
-    return ((int)mem_.getCapacity() - COMBINED_BUFFER) / 8;
+    return Math.max(0, (int)mem_.getCapacity() - COMBINED_BUFFER) / 8;
   }
 
   @Override
diff --git a/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java
index 2fc399f9b..4c9f74286 100644
--- a/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java
@@ -460,9 +460,9 @@ public static int getUpdatableStorageBytes(final int k, final long n) {
     final int totLevels = computeNumLevelsNeeded(k, n);
     if (n <= k) {
       final int ceil = Math.max(ceilingPowerOf2((int)n), MIN_K * 2);
-      return metaPre + ceil << 3;
+      return (metaPre + ceil) << 3;
     }
-    return metaPre + (2 + totLevels) * k << 3;
+    return (metaPre + (2 + totLevels) * k) << 3;
   }
 
   /**
diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java
index 946603194..dbdedebd5 100644
--- a/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java
@@ -199,7 +199,7 @@ public void awaitBgPropagationTermination() {
 
   @Override
   public final void initBgPropagationService() {
-    executorService_ = ConcurrentPropagationService.getExecutorService(Thread.currentThread().getId());
+    executorService_ = ConcurrentPropagationService.getExecutorService(Thread.currentThread().threadId());
   }
 
   @Override
@@ -258,7 +258,7 @@ public boolean validateEpoch(final long epoch) {
   private void advanceEpoch() {
     awaitBgPropagationTermination();
     startEagerPropagation();
-    ConcurrentPropagationService.resetExecutorService(Thread.currentThread().getId());
+    ConcurrentPropagationService.resetExecutorService(Thread.currentThread().threadId());
     //no inspection NonAtomicOperationOnVolatileField
     // this increment of a volatile field is done within the scope of the propagation
     // synchronization and hence is done by a single thread.
diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketch.java
index 1ce3b4ecc..e4cc3a157 100644
--- a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketch.java
@@ -194,7 +194,7 @@ public void awaitBgPropagationTermination() {
 
   @Override
   public void initBgPropagationService() {
-    executorService_ = ConcurrentPropagationService.getExecutorService(Thread.currentThread().getId());
+    executorService_ = ConcurrentPropagationService.getExecutorService(Thread.currentThread().threadId());
   }
 
   @Override
@@ -253,7 +253,7 @@ public boolean validateEpoch(final long epoch) {
   private void advanceEpoch() {
     awaitBgPropagationTermination();
     startEagerPropagation();
-    ConcurrentPropagationService.resetExecutorService(Thread.currentThread().getId());
+    ConcurrentPropagationService.resetExecutorService(Thread.currentThread().threadId());
     //no inspection NonAtomicOperationOnVolatileField
     // this increment of a volatile field is done within the scope of the propagation
     // synchronization and hence is done by a single thread
diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java
index ad9051a08..af073a5ee 100644
--- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java
@@ -327,11 +327,11 @@ UpdateReturnState hashUpdate(final long hash) {
             throw new SketchesArgumentException("Out of Memory, MemoryRequestServer is null, cannot expand.");
           }
 
-          final WritableMemory newDstMem = memReqSvr_.request(wmem_,reqBytes);
+          final WritableMemory newDstMem = memReqSvr_.request(wmem_, reqBytes);
 
           moveAndResize(wmem_, preambleLongs, lgArrLongs, newDstMem, tgtLgArrLongs, thetaLong);
 
-          memReqSvr_.requestClose(wmem_, newDstMem);
+          memReqSvr_.requestClose(wmem_);
 
           wmem_ = newDstMem;
           hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, tgtLgArrLongs);
diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java
index 7d72fae2f..2d71805ac 100644
--- a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java
+++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java
@@ -31,7 +31,7 @@
 import org.apache.datasketches.memory.WritableMemory;
 import org.testng.annotations.Test;
 
-import jdk.incubator.foreign.ResourceScope;
+import java.lang.foreign.Arena;
 
 public class BloomFilterTest {
 
@@ -52,8 +52,9 @@ public void createNewFilterTest() throws Exception {
     assertFalse(bf1.isDirect());
     assertFalse(bf1.isReadOnly());
 
-    WritableMemory wmem;
-    try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(sizeBytes)).scope()) {
+
+    try (Arena arena = Arena.ofConfined()) {
+      WritableMemory wmem = WritableMemory.allocateDirect(sizeBytes, arena);
       final BloomFilter bf2 = new BloomFilter(numBits, numHashes, seed, wmem);
       assertTrue(bf2.isEmpty());
       assertTrue(bf2.hasMemory());
@@ -158,8 +159,9 @@ public void basicFilterOperationsTest() {
 
     int numFound = 0;
     for (long i = 0; i < 2 * n; ++i) {
-      if (bf.query(i))
+      if (bf.query(i)) {
         ++numFound;
+      }
     }
     assertTrue(numFound >= n);
     assertTrue(numFound < 1.1 * n);
@@ -339,8 +341,12 @@ public void nonEmptySerializationTest() {
     int fromBytesCount = 0;
     for (int i = 0; i < numBits; ++i) {
       boolean val = fromBytes.query(0.5 + i);
-      if (val) ++fromBytesCount;
-      if (i < n) assertTrue(val);
+      if (val) {
+        ++fromBytesCount;
+      }
+      if (i < n) {
+        assertTrue(val);
+      }
     }
     assertEquals(fromBytesCount, n + count); // same numbers of items should match
 
@@ -354,8 +360,12 @@ public void nonEmptySerializationTest() {
     int fromLongsCount = 0;
     for (int i = 0; i < numBits; ++i) {
       boolean val = fromLongs.query(0.5 + i);
-      if (val) ++fromLongsCount;
-      if (i < n) assertTrue(val);
+      if (val) {
+        ++fromLongsCount;
+      }
+      if (i < n) {
+        assertTrue(val);
+      }
     }
     assertEquals(fromLongsCount, n + count); // same numbers of items should match
 
diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3AdaptorTest.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3AdaptorTest.java
deleted file mode 100644
index 16a8e6652..000000000
--- a/src/test/java/org/apache/datasketches/hash/MurmurHash3AdaptorTest.java
+++ /dev/null
@@ -1,377 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.datasketches.hash;
-
-import static org.apache.datasketches.hash.MurmurHash3Adaptor.asDouble;
-import static org.apache.datasketches.hash.MurmurHash3Adaptor.asInt;
-import static org.apache.datasketches.hash.MurmurHash3Adaptor.hashToBytes;
-import static org.apache.datasketches.hash.MurmurHash3Adaptor.hashToLongs;
-import static org.apache.datasketches.hash.MurmurHash3Adaptor.modulo;
-
-import org.apache.datasketches.common.SketchesArgumentException;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-/**
- * @author Lee Rhodes
- */
-public class MurmurHash3AdaptorTest {
-
-  @Test
-  public void checkToBytesLong() {
-    byte[] result = hashToBytes(2L, 0L);
-    for (int i = 8; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-  }
-
-  @Test
-  public void checkToBytesLongArr() {
-    long[] arr = { 1L, 2L };
-    byte[] result = hashToBytes(arr, 0L);
-    for (int i = 8; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    arr = null;
-    result = hashToBytes(arr, 0L);
-    Assert.assertEquals(result, null);
-
-    arr = new long[0];
-    result = hashToBytes(arr, 0L);
-    Assert.assertEquals(result, null);
-  }
-
-  @Test
-  public void checkToBytesIntArr() {
-    int[] arr = { 1, 2 };
-    byte[] result = hashToBytes(arr, 0L);
-    for (int i = 8; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    arr = null;
-    result = hashToBytes(arr, 0L);
-    Assert.assertEquals(result, null);
-
-    arr = new int[0];
-    result = hashToBytes(arr, 0L);
-    Assert.assertEquals(result, null);
-  }
-
-  @Test
-  public void checkToBytesCharArr() {
-    char[] arr = { 1, 2 };
-    byte[] result = hashToBytes(arr, 0L);
-    for (int i = 8; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    arr = null;
-    result = hashToBytes(arr, 0L);
-    Assert.assertEquals(result, null);
-
-    arr = new char[0];
-    result = hashToBytes(arr, 0L);
-    Assert.assertEquals(result, null);
-  }
-
-  @Test
-  public void checkToBytesByteArr() {
-    byte[] arr = { 1, 2 };
-    byte[] result = hashToBytes(arr, 0L);
-    for (int i = 8; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    arr = null;
-    result = hashToBytes(arr, 0L);
-    Assert.assertEquals(result, null);
-
-    arr = new byte[0];
-    result = hashToBytes(arr, 0L);
-    Assert.assertEquals(result, null);
-
-  }
-
-  @Test
-  public void checkToBytesDouble() {
-    byte[] result = hashToBytes(1.0, 0L);
-    for (int i = 8; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    result = hashToBytes(0.0, 0L);
-    for (int i = 8; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    result = hashToBytes( -0.0, 0L);
-    for (int i = 8; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-  }
-
-  @Test
-  public void checkToBytesString() {
-    byte[] result = hashToBytes("1", 0L);
-    for (int i = 8; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    result = hashToBytes("", 0L);
-    Assert.assertEquals(result, null);
-
-    String s = null;
-    result = hashToBytes(s, 0L);
-    Assert.assertEquals(result, null);
-  }
-
-  /************/
-
-  @Test
-  public void checkToLongsLong() {
-    long[] result = hashToLongs(2L, 0L);
-    for (int i = 2; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-  }
-
-  @Test
-  public void checkToLongsLongArr() {
-    long[] arr = { 1L, 2L };
-    long[] result = hashToLongs(arr, 0L);
-    for (int i = 2; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    arr = null;
-    result = hashToLongs(arr, 0L);
-    Assert.assertEquals(result, null);
-
-    arr = new long[0];
-    result = hashToLongs(arr, 0L);
-    Assert.assertEquals(result, null);
-  }
-
-  @Test
-  public void checkToLongsIntArr() {
-    int[] arr = { 1, 2 };
-    long[] result = hashToLongs(arr, 0L);
-    for (int i = 2; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    arr = null;
-    result = hashToLongs(arr, 0L);
-    Assert.assertEquals(result, null);
-
-    arr = new int[0];
-    result = hashToLongs(arr, 0L);
-    Assert.assertEquals(result, null);
-  }
-
-  @Test
-  public void checkToLongsCharArr() {
-    char[] arr = { 1, 2 };
-    long[] result = hashToLongs(arr, 0L);
-    for (int i = 2; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    arr = null;
-    result = hashToLongs(arr, 0L);
-    Assert.assertEquals(result, null);
-
-    arr = new char[0];
-    result = hashToLongs(arr, 0L);
-    Assert.assertEquals(result, null);
-  }
-
-  @Test
-  public void checkToLongsByteArr() {
-    byte[] arr = { 1, 2 };
-    long[] result = hashToLongs(arr, 0L);
-    for (int i = 2; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    arr = null;
-    result = hashToLongs(arr, 0L);
-    Assert.assertEquals(result, null);
-
-    arr = new byte[0];
-    result = hashToLongs(arr, 0L);
-    Assert.assertEquals(result, null);
-
-  }
-
-  @Test
-  public void checkToLongsDouble() {
-    long[] result = hashToLongs(1.0, 0L);
-    for (int i = 2; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    result = hashToLongs(0.0, 0L);
-    for (int i = 2; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    result = hashToLongs( -0.0, 0L);
-    for (int i = 2; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-  }
-
-  @Test
-  public void checkToLongsString() {
-    long[] result = hashToLongs("1", 0L);
-    for (int i = 2; i-- > 0;) {
-      Assert.assertNotEquals(result[i], 0);
-    }
-    result = hashToLongs("", 0L);
-    Assert.assertEquals(result, null);
-    String s = null;
-    result = hashToLongs(s, 0L);
-    Assert.assertEquals(result, null);
-  }
-
-  /*************/
-
-  @Test
-  public void checkModulo() {
-    int div = 7;
-    for (int i = 20; i-- > 0;) {
-      long[] out = hashToLongs(i, 9001);
-      int mod = modulo(out[0], out[1], div);
-      Assert.assertTrue((mod < div) && (mod >= 0));
-      mod = modulo(out, div);
-      Assert.assertTrue((mod < div) && (mod >= 0));
-    }
-  }
-
-  @Test
-  public void checkAsDouble() {
-    for (int i = 0; i < 10000; i++ ) {
-      double result = asDouble(hashToLongs(i, 0));
-      Assert.assertTrue((result >= 0) && (result < 1.0));
-    }
-  }
-
-  //Check asInt() functions
-
-  @Test
-  public void checkAsInt() {
-    int lo = (3 << 28);
-    int hi = (1 << 30) + 1;
-    for (byte i = 0; i < 126; i++ ) {
-      long[] longArr = {i, i+1};         //long[]
-      int result = asInt(longArr, lo);
-      Assert.assertTrue((result >= 0) && (result < lo));
-      result = asInt(longArr, hi);
-      Assert.assertTrue((result >= 0) && (result < hi));
-
-      int[] intArr = {i, i+1};           //int[]
-      result = asInt(intArr, lo);
-      Assert.assertTrue((result >= 0) && (result < lo));
-      result = asInt(intArr, hi);
-      Assert.assertTrue((result >= 0) && (result < hi));
-
-      byte[] byteArr = {i, (byte)(i+1)}; //byte[]
-      result = asInt(byteArr, lo);
-      Assert.assertTrue((result >= 0) && (result < lo));
-      result = asInt(byteArr, hi);
-      Assert.assertTrue((result >= 0) && (result < hi));
-
-      long longV = i;                    //long
-      result = asInt(longV, lo);
-      Assert.assertTrue((result >= 0) && (result < lo));
-      result = asInt(longV, hi);
-      Assert.assertTrue((result >= 0) && (result < hi));
-
-      double v = i;                //double
-      result = asInt(v, lo);
-      Assert.assertTrue((result >= 0) && (result < lo));
-      result = asInt(v, hi);
-      Assert.assertTrue((result >= 0) && (result < hi));
-
-      String s = Integer.toString(i);    //String
-      result = asInt(s, lo);
-      Assert.assertTrue((result >= 0) && (result < lo));
-      result = asInt(s, hi);
-      Assert.assertTrue((result >= 0) && (result < hi));
-    }
-  }
-
-  @Test (expectedExceptions = SketchesArgumentException.class)
-  public void checkAsIntCornerCaseLongNull() {
-    long[] arr = null;
-    asInt(arr, 1000);
-  }
-
-  @Test (expectedExceptions = SketchesArgumentException.class)
-  public void checkAsIntCornerCaseLongEmpty() {
-    long[] arr = new long[0];
-    asInt(arr, 1000);
-  }
-
-  @Test (expectedExceptions = SketchesArgumentException.class)
-  public void checkAsIntCornerCaseIntNull() {
-    int[] arr = null;
-    asInt(arr, 1000);
-  }
-
-  @Test (expectedExceptions = SketchesArgumentException.class)
-  public void checkAsIntCornerCaseIntEmpty() {
-    int[] arr = new int[0];
-    asInt(arr, 1000);
-  }
-
-  @Test (expectedExceptions = SketchesArgumentException.class)
-  public void checkAsIntCornerCaseByteNull() {
-    byte[] arr = null;
-    asInt(arr, 1000);
-  }
-
-  @Test (expectedExceptions = SketchesArgumentException.class)
-  public void checkAsIntCornerCaseByteEmpty() {
-    byte[] arr = new byte[0];
-    asInt(arr, 1000);
-  }
-
-  @Test (expectedExceptions = SketchesArgumentException.class)
-  public void checkAsIntCornerCaseStringNull() {
-    String s = null;
-    asInt(s, 1000);
-  }
-
-  @Test (expectedExceptions = SketchesArgumentException.class)
-  public void checkAsIntCornerCaseStringEmpty() {
-    String s = "";
-    asInt(s, 1000);
-  }
-
-  @Test (expectedExceptions = SketchesArgumentException.class)
-  public void checkAsIntCornerCaseNTooSmall() {
-    String s = "abc";
-    asInt(s, 1);
-  }
-
-  @Test
-  public void printlnTest() {
-    println("PRINTING: "+this.getClass().getName());
-  }
-
-  /**
-   * @param s value to print
-   */
-  static void println(String s) {
-    //System.out.println(s); //disable here
-  }
-
-}
diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3v4Test.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21Test.java
similarity index 83%
rename from src/test/java/org/apache/datasketches/hash/MurmurHash3v4Test.java
rename to src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21Test.java
index 6cd4d624f..d9771d304 100644
--- a/src/test/java/org/apache/datasketches/hash/MurmurHash3v4Test.java
+++ b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21Test.java
@@ -28,13 +28,12 @@
 import org.testng.annotations.Test;
 
 import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.memory.internal.MurmurHash3v4;
 import org.apache.datasketches.memory.WritableMemory;
 
 /**
  * @author Lee Rhodes
  */
-public class MurmurHash3v4Test {
+public class MurmurHash3FFM21Test {
   private Random rand = new Random();
   private static final int trials = 1 << 20;
 
@@ -154,25 +153,25 @@ private static final long[] hashV1(byte[] key, long seed) {
   }
 
   private static final long[] hashV2(long[] key, long seed) {
-    return MurmurHash3v4.hash(key, seed);
+    return MurmurHash3FFM21.hash(key, seed);
   }
 
   private static final long[] hashV2(int[] key2, long seed) {
-    return MurmurHash3v4.hash(key2, seed);
+    return MurmurHash3FFM21.hash(key2, seed);
   }
 
   private static final long[] hashV2(char[] key, long seed) {
-    return MurmurHash3v4.hash(key, seed);
+    return MurmurHash3FFM21.hash(key, seed);
   }
 
   private static final long[] hashV2(byte[] key, long seed) {
-    return MurmurHash3v4.hash(key, seed);
+    return MurmurHash3FFM21.hash(key, seed);
   }
 
   //V2 single primitives
 
   private static final long[] hashV2(long key, long seed, long[] out) {
-    return MurmurHash3v4.hash(key, seed, out);
+    return MurmurHash3FFM21.hash(key, seed, out);
   }
 
 //  private static final long[] hashV2(double key, long seed, long[] out) {
@@ -183,8 +182,6 @@ private static final long[] hashV2(long key, long seed, long[] out) {
 //    return MurmurHash3v4.hash(key, seed, out);
 //  }
 
-
-
   @Test
   public void offsetChecks() {
     long seed = 12345;
@@ -197,9 +194,9 @@ public void offsetChecks() {
     WritableMemory wmem = WritableMemory.allocate(cap);
     for (int i = 0; i < cap; i++) { wmem.putByte(i, (byte)(-128 + i)); }
 
-    for (int offset = 3; offset < 16; offset++) {
+    for (int offset = 0; offset < 16; offset++) {
       int arrLen = cap - offset;
-      hash1 = MurmurHash3v4.hash(wmem, offset, arrLen, seed, hash1);
+      hash1 = MurmurHash3FFM21.hash(wmem, offset, arrLen, seed, hash1);
       byte[] byteArr2 = new byte[arrLen];
       wmem.getByteArray(offset, byteArr2, 0, arrLen);
       hash2 = MurmurHash3.hash(byteArr2, seed);
@@ -222,8 +219,8 @@ public void byteArrChecks() {
       for (int i = 0; i < j; i++) { wmem.putByte(i, (byte) (-128 + i)); }
 
       long[] hash1 = MurmurHash3.hash(in, 0);
-      hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2);
-      long[] hash3 = MurmurHash3v4.hash(in, seed);
+      hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2);
+      long[] hash3 = MurmurHash3FFM21.hash(in, seed);
 
       assertEquals(hash1, hash2);
       assertEquals(hash1, hash3);
@@ -246,8 +243,8 @@ public void charArrChecks() {
       for (int i = 0; i < j; i++) { wmem.putInt(i, i); }
 
       long[] hash1 = MurmurHash3.hash(in, 0);
-      hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2);
-      long[] hash3 = MurmurHash3v4.hash(in, seed);
+      hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2);
+      long[] hash3 = MurmurHash3FFM21.hash(in, seed);
 
       assertEquals(hash1, hash2);
       assertEquals(hash1, hash3);
@@ -270,8 +267,8 @@ public void intArrChecks() {
       for (int i = 0; i < j; i++) { wmem.putInt(i, i); }
 
       long[] hash1 = MurmurHash3.hash(in, 0);
-      hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2);
-      long[] hash3 = MurmurHash3v4.hash(in, seed);
+      hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2);
+      long[] hash3 = MurmurHash3FFM21.hash(in, seed);
 
       assertEquals(hash1, hash2);
       assertEquals(hash1, hash3);
@@ -294,8 +291,8 @@ public void longArrChecks() {
       for (int i = 0; i < j; i++) { wmem.putLong(i, i); }
 
       long[] hash1 = MurmurHash3.hash(in, 0);
-      hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2);
-      long[] hash3 = MurmurHash3v4.hash(in, seed);
+      hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2);
+      long[] hash3 = MurmurHash3FFM21.hash(in, seed);
 
       assertEquals(hash1, hash2);
       assertEquals(hash1, hash3);
@@ -313,8 +310,8 @@ public void longCheck() {
     WritableMemory wmem = WritableMemory.writableWrap(in);
 
     long[] hash1 = MurmurHash3.hash(in, 0);
-    hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2);
-    long[] hash3 = MurmurHash3v4.hash(in, seed);
+    hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2);
+    long[] hash3 = MurmurHash3FFM21.hash(in, seed);
 
     assertEquals(hash1, hash2);
     assertEquals(hash1, hash3);
@@ -325,57 +322,57 @@ public void checkEmptiesNulls() {
     long seed = 123;
     long[] hashOut = new long[2];
     try {
-      MurmurHash3v4.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut);  //mem empty
+      MurmurHash3FFM21.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut);  //mem empty
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       String s = "";
-      MurmurHash3v4.hash(s, seed, hashOut); //string empty
+      MurmurHash3FFM21.hash(s, seed, hashOut); //string empty
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       String s = null;
-      MurmurHash3v4.hash(s, seed, hashOut); //string null
+      MurmurHash3FFM21.hash(s, seed, hashOut); //string null
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       byte[] barr = new byte[0];
-      MurmurHash3v4.hash(barr, seed); //byte[] empty
+      MurmurHash3FFM21.hash(barr, seed); //byte[] empty
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       byte[] barr = null;
-      MurmurHash3v4.hash(barr, seed); //byte[] null
+      MurmurHash3FFM21.hash(barr, seed); //byte[] null
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       char[] carr = new char[0];
-      MurmurHash3v4.hash(carr, seed); //char[] empty
+      MurmurHash3FFM21.hash(carr, seed); //char[] empty
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       char[] carr = null;
-      MurmurHash3v4.hash(carr, seed); //char[] null
+      MurmurHash3FFM21.hash(carr, seed); //char[] null
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       int[] iarr = new int[0];
-      MurmurHash3v4.hash(iarr, seed); //int[] empty
+      MurmurHash3FFM21.hash(iarr, seed); //int[] empty
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       int[] iarr = null;
-      MurmurHash3v4.hash(iarr, seed); //int[] null
+      MurmurHash3FFM21.hash(iarr, seed); //int[] null
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       long[] larr = new long[0];
-      MurmurHash3v4.hash(larr, seed); //long[] empty
+      MurmurHash3FFM21.hash(larr, seed); //long[] empty
       fail();
     } catch (final IllegalArgumentException e) { } //OK
     try {
       long[] larr = null;
-      MurmurHash3v4.hash(larr, seed); //long[] null
+      MurmurHash3FFM21.hash(larr, seed); //long[] null
       fail();
     } catch (final IllegalArgumentException e) { } //OK
   }
@@ -385,9 +382,9 @@ public void checkStringLong() {
     long seed = 123;
     long[] hashOut = new long[2];
     String s = "123";
-    assertTrue(MurmurHash3v4.hash(s, seed, hashOut)[0] != 0);
+    assertTrue(MurmurHash3FFM21.hash(s, seed, hashOut)[0] != 0);
     long v = 123;
-    assertTrue(MurmurHash3v4.hash(v, seed, hashOut)[0] != 0);
+    assertTrue(MurmurHash3FFM21.hash(v, seed, hashOut)[0] != 0);
   }
 
   @Test
@@ -415,8 +412,8 @@ private static long[] checkDouble(double dbl) {
 
     WritableMemory wmem = WritableMemory.writableWrap(dataArr);
     long[] hash1 = MurmurHash3.hash(dataArr, 0);
-    hash2 = MurmurHash3v4.hash(wmem, offset, bytes, seed, hash2);
-    long[] hash3 = MurmurHash3v4.hash(dbl, seed, hash2);
+    hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2);
+    long[] hash3 = MurmurHash3FFM21.hash(dbl, seed, hash2);
 
     assertEquals(hash1, hash2);
     assertEquals(hash1, hash3);
diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java
new file mode 100644
index 000000000..1e23772b5
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java
@@ -0,0 +1,419 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hash;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.datasketches.hash.MurmurHash3FFM21.hash;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.nio.ByteOrder;
+
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.MemoryRequestServer;
+import org.apache.datasketches.memory.Resource;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/**
+ * Tests the MurmurHash3 against specific, known hash results given known
+ * inputs obtained from the public domain C++ version 150.
+ *
+ * @author Lee Rhodes
+ */
+public class MurmurHash3FFM21bTest {
+  private static final MemoryRequestServer memReqSvr = Resource.defaultMemReqSvr;
+
+  @Test
+  public void checkByteArrRemainderGT8() { //byte[], remainder > 8
+    String keyStr = "The quick brown fox jumps over the lazy dog";
+    byte[] key = keyStr.getBytes(UTF_8);
+    long[] result = hash(key, 0);
+    //Should be:
+    long h1 = 0xe34bbc7bbc071b6cL;
+    long h2 = 0x7a433ca9c49a9347L;
+    Assert.assertEquals(result[0], h1);
+    Assert.assertEquals(result[1], h2);
+  }
+
+  @Test
+  public void checkByteArrRemainderGT8withSegment() { //byte[], remainder > 8
+    String keyStr = "The quick brown fox jumps over the lazy dog";
+    byte[] key = keyStr.getBytes(UTF_8);
+    long[] out = new long[2];
+    MemorySegment seg = MemorySegment.ofArray(key);
+    long[] result = hash(seg, 0, seg.byteSize(), 0, out);
+    //Should be:
+    long h1 = 0xe34bbc7bbc071b6cL;
+    long h2 = 0x7a433ca9c49a9347L;
+    Assert.assertEquals(result[0], h1);
+    Assert.assertEquals(result[1], h2);
+  }
+
+  @Test
+  public void checkByteArrChange1bit() { //byte[], change one bit
+    String keyStr = "The quick brown fox jumps over the lazy eog";
+    byte[] key = keyStr.getBytes(UTF_8);
+    long[] result = hash(key, 0);
+    //Should be:
+    long h1 = 0x362108102c62d1c9L;
+    long h2 = 0x3285cd100292b305L;
+    Assert.assertEquals(result[0], h1);
+    Assert.assertEquals(result[1], h2);
+  }
+
+  @Test
+  public void checkByteArrRemainderLt8() { //byte[], test a remainder < 8
+    String keyStr = "The quick brown fox jumps over the lazy dogdogdog";
+    byte[] key = keyStr.getBytes(UTF_8);
+    long[] result = hash(key, 0);
+    //Should be;
+    long h1 = 0x9c8205300e612fc4L;
+    long h2 = 0xcbc0af6136aa3df9L;
+    Assert.assertEquals(result[0], h1);
+    Assert.assertEquals(result[1], h2);
+  }
+
+  @Test
+  public void checkByteArrReaminderEQ8() { //byte[], test a remainder = 8
+    String keyStr = "The quick brown fox jumps over the lazy1";
+    byte[] key = keyStr.getBytes(UTF_8);
+    long[] result = hash(key, 0);
+    //Should be:
+    long h1 = 0xe3301a827e5cdfe3L;
+    long h2 = 0xbdbf05f8da0f0392L;
+    Assert.assertEquals(result[0], h1);
+    Assert.assertEquals(result[1], h2);
+  }
+
+  /**
+   * This test should have the exact same output as Test4
+   */
+  @Test
+  public void checkLongArrRemainderEQ8() { //long[], test a remainder = 8
+    String keyStr = "The quick brown fox jumps over the lazy1";
+    long[] key = stringToLongs(keyStr);
+    long[] result = hash(key, 0);
+    //Should be:
+    long h1 = 0xe3301a827e5cdfe3L;
+    long h2 = 0xbdbf05f8da0f0392L;
+    Assert.assertEquals(result[0], h1);
+    Assert.assertEquals(result[1], h2);
+  }
+
+  /**
+   * This test should have the exact same output as Test4
+   */
+  @Test
+  public void checkIntArrRemainderEQ8() { //int[], test a remainder = 8
+    String keyStr = "The quick brown fox jumps over the lazy1"; //40B
+    int[] key = stringToInts(keyStr);
+    long[] result = hash(key, 0);
+    //Should be:
+    long h1 = 0xe3301a827e5cdfe3L;
+    long h2 = 0xbdbf05f8da0f0392L;
+    Assert.assertEquals(result[0], h1);
+    Assert.assertEquals(result[1], h2);
+  }
+
+  @Test
+  public void checkIntArrRemainderEQ0() { //int[], test a remainder = 0
+    String keyStr = "The quick brown fox jumps over t"; //32B
+    int[] key = stringToInts(keyStr);
+    long[] result = hash(key, 0);
+    //Should be:
+    long h1 = 0xdf6af91bb29bdacfL;
+    long h2 = 0x91a341c58df1f3a6L;
+    Assert.assertEquals(result[0], h1);
+    Assert.assertEquals(result[1], h2);
+  }
+
+  /**
+   * Tests an odd remainder of int[].
+   */
+  @Test
+  public void checkIntArrOddRemainder() { //int[], odd remainder
+    String keyStr = "The quick brown fox jumps over the lazy dog"; //43B
+    int[] key = stringToInts(keyStr);
+    long[] result = hash(key, 0);
+    //Should be:
+    long h1 = 0x1eb232b0087543f5L;
+    long h2 = 0xfc4c1383c3ace40fL;
+    Assert.assertEquals(result[0], h1);
+    Assert.assertEquals(result[1], h2);
+  }
+
+  /**
+   * Tests an odd remainder of int[].
+   */
+  @Test
+  public void checkCharArrOddRemainder() { //char[], odd remainder
+    String keyStr = "The quick brown fox jumps over the lazy dog.."; //45B
+    char[] key = keyStr.toCharArray();
+    long[] result = hash(key, 0);
+    //Should be:
+    long h1 = 0xca77b498ea9ed953L;
+    long h2 = 0x8b8f8ec3a8f4657eL;
+    Assert.assertEquals(result[0], h1);
+    Assert.assertEquals(result[1], h2);
+  }
+
+  /**
+   * Tests an odd remainder of int[].
+   */
+  @Test
+  public void checkCharArrRemainderEQ0() { //char[], remainder of 0
+    String keyStr = "The quick brown fox jumps over the lazy "; //40B
+    char[] key = keyStr.toCharArray();
+    long[] result = hash(key, 0);
+    //Should be:
+    long h1 = 0x51b15e9d0887f9f1L;
+    long h2 = 0x8106d226786511ebL;
+    Assert.assertEquals(result[0], h1);
+    Assert.assertEquals(result[1], h2);
+  }
+
+  @Test
+  public void checkByteArrAllOnesZeros() { //byte[], test a ones byte and a zeros byte
+    byte[] key = {
+      0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69, 0x63, 0x6b, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x6e,
+      0x20, 0x66, 0x6f, 0x78, 0x20, 0x6a, 0x75, 0x6d, 0x70, 0x73, 0x20, 0x6f, 0x76, 0x65,
+      0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6c, 0x61, 0x7a, 0x79, 0x20, 0x64, 0x6f, 0x67,
+      (byte) 0xff, 0x64, 0x6f, 0x67, 0x00
+    };
+    long[] result = MurmurHash3FFM21.hash(key, 0);
+    //Should be:
+    long h1 = 0xe88abda785929c9eL;
+    long h2 = 0x96b98587cacc83d6L;
+    Assert.assertEquals(result[0], h1);
+    Assert.assertEquals(result[1], h2);
+  }
+
+  /**
+   * This test demonstrates that the hash of byte[], char[], int[], or long[] will produce the
+   * same hash result if, and only if, all the arrays have the same exact length in bytes, and if
+   * the contents of the values in the arrays have the same byte endianness and overall order.
+   */
+  @Test
+  public void checkCrossTypeHashConsistency() {
+    long[] out;
+    println("Bytes");
+    byte[] bArr = {1,2,3,4,5,6,7,8,   9,10,11,12,13,14,15,16,  17,18,19,20,21,22,23,24};
+    long[] out1 = hash(bArr, 0L);
+    println(longToHexBytes(out1[0]));
+    println(longToHexBytes(out1[1]));
+
+    println("Chars");
+    char[] cArr = {0X0201, 0X0403, 0X0605, 0X0807,   0X0a09, 0X0c0b, 0X0e0d, 0X100f,
+        0X1211, 0X1413, 0X1615, 0X1817};
+    out = hash(cArr, 0L);
+    Assert.assertEquals(out, out1);
+    println(longToHexBytes(out[0]));
+    println(longToHexBytes(out[1]));
+
+    println("Ints");
+    int[] iArr = {0X04030201, 0X08070605,   0X0c0b0a09, 0X100f0e0d,   0X14131211,   0X18171615};
+    out = hash(iArr, 0L);
+    Assert.assertEquals(out, out1);
+    println(longToHexBytes(out[0]));
+    println(longToHexBytes(out[1]));
+
+    println("Longs");
+    long[] lArr = {0X0807060504030201L, 0X100f0e0d0c0b0a09L, 0X1817161514131211L};
+    out = hash(lArr, 0L);
+    Assert.assertEquals(out, out1);
+    println(longToHexBytes(out[0]));
+    println(longToHexBytes(out[1]));
+  }
+
+  @Test
+  public void checkEmptyOrNullExceptions() {
+    try {
+      long[] arr = null;
+      hash(arr, 1L);
+      fail();
+    }
+    catch (final IllegalArgumentException e) { }
+    try {
+      int[] arr = null; hash(arr, 1L); fail();
+    } catch (final IllegalArgumentException e) { }
+    try {
+      char[] arr = null; hash(arr, 1L); fail();
+    } catch (final IllegalArgumentException e) { }
+    try {
+      byte[] arr = null; hash(arr, 1L); fail();
+    } catch (final IllegalArgumentException e) { }
+
+    long[] out = new long[2];
+    try {
+      String in = null; hash(in, 1L, out); fail();
+    } catch (final IllegalArgumentException e) { }
+    try {
+      Memory mem = Memory.wrap(new byte[0]);
+      hash(mem, 0L, 4L, 1L, out);
+    } catch (final IllegalArgumentException e) { }
+    try (Arena arena = Arena.ofConfined()) {
+      Memory mem = WritableMemory.allocateDirect(8, 1, ByteOrder.nativeOrder(), memReqSvr, arena);
+      out = hash(mem, 0L, 4L, 1L, out);
+    }
+    assertTrue((out[0] != 0) && (out[1] != 0));
+  }
+
+  @Test
+  public void checkHashTails() {
+    long[] out = new long[2];
+    WritableMemory mem = WritableMemory.allocate(32);
+    mem.fill((byte)85);
+
+    for (int i = 16; i <= 32; i++) {
+      out = hash(mem, 0, i, 1L, out);
+    }
+  }
+
+  @Test
+  public void checkSinglePrimitives() {
+    long[] out = new long[2];
+    out = hash(1L, 1L, out);
+    out = hash(0.0, 1L, out);
+    out = hash("123", 1L, out);
+  }
+
+  //Helper methods
+
+  private static long[] stringToLongs(String in) {
+    byte[] bArr = in.getBytes(UTF_8);
+    int inLen = bArr.length;
+    int outLen = (inLen / 8) + (((inLen % 8) != 0) ? 1 : 0);
+    long[] out = new long[outLen];
+
+    for (int i = 0; i < (outLen - 1); i++ ) {
+      for (int j = 0; j < 8; j++ ) {
+        out[i] |= ((bArr[(i * 8) + j] & 0xFFL) << (j * 8));
+      }
+    }
+    int inTail = 8 * (outLen - 1);
+    int rem = inLen - inTail;
+    for (int j = 0; j < rem; j++ ) {
+      out[outLen - 1] |= ((bArr[inTail + j] & 0xFFL) << (j * 8));
+    }
+    return out;
+  }
+
+  private static int[] stringToInts(String in) {
+    byte[] bArr = in.getBytes(UTF_8);
+    int inLen = bArr.length;
+    int outLen = (inLen / 4) + (((inLen % 4) != 0) ? 1 : 0);
+    int[] out = new int[outLen];
+
+    for (int i = 0; i < (outLen - 1); i++ ) {
+      for (int j = 0; j < 4; j++ ) {
+        out[i] |= ((bArr[(i * 4) + j] & 0xFFL) << (j * 8));
+      }
+    }
+    int inTail = 4 * (outLen - 1);
+    int rem = inLen - inTail;
+    for (int j = 0; j < rem; j++ ) {
+      out[outLen - 1] |= ((bArr[inTail + j] & 0xFFL) << (j * 8));
+    }
+    return out;
+  }
+
+  /**
+   * Returns a string of spaced hex bytes in Big-Endian order.
+   * @param v the given long
+   * @return string of spaced hex bytes in Big-Endian order.
+   */
+  private static String longToHexBytes(final long v) {
+    final long mask = 0XFFL;
+    final StringBuilder sb = new StringBuilder();
+    for (int i = 8; i-- > 0; ) {
+      final String s = Long.toHexString((v >>> (i * 8)) & mask);
+      sb.append(zeroPad(s, 2)).append(" ");
+    }
+    return sb.toString();
+  }
+
+  /**
+   * Prepend the given string with zeros. If the given string is equal or greater than the given
+   * field length, it will be returned without modification.
+   * @param s the given string
+   * @param fieldLength desired total field length including the given string
+   * @return the given string prepended with zeros.
+   */
+  private static final String zeroPad(final String s, final int fieldLength) {
+    return characterPad(s, fieldLength, '0', false);
+  }
+
+  /**
+   * Prepend or postpend the given string with the given character to fill the given field length.
+   * If the given string is equal or greater than the given field length, it will be returned
+   * without modification.
+   * @param s the given string
+   * @param fieldLength the desired field length
+   * @param padChar the desired pad character
+   * @param postpend if true append the padCharacters to the end of the string.
+   * @return prepended or postpended given string with the given character to fill the given field
+   * length.
+   */
+  private static final String characterPad(final String s, final int fieldLength, final char padChar,
+      final boolean postpend) {
+    final char[] chArr = s.toCharArray();
+    final int sLen = chArr.length;
+    if (sLen < fieldLength) {
+      final char[] out = new char[fieldLength];
+      final int blanks = fieldLength - sLen;
+
+      if (postpend) {
+        for (int i = 0; i < sLen; i++) {
+          out[i] = chArr[i];
+        }
+        for (int i = sLen; i < fieldLength; i++) {
+          out[i] = padChar;
+        }
+      } else { //prepend
+        for (int i = 0; i < blanks; i++) {
+          out[i] = padChar;
+        }
+        for (int i = blanks; i < fieldLength; i++) {
+          out[i] = chArr[i - blanks];
+        }
+      }
+
+      return String.valueOf(out);
+    }
+    return s;
+  }
+
+  @Test
+  public void printlnTest() {
+    println("PRINTING: "+this.getClass().getName());
+  }
+
+  /**
+   * @param s value to print
+   */
+  static void println(String s) {
+    //System.out.println(s); //disable here
+  }
+
+}
diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
index 75504bc83..ecebe4539 100644
--- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
+++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
@@ -26,11 +26,10 @@
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 
-import java.nio.ByteOrder;
+import java.lang.foreign.Arena;
 import java.util.HashMap;
 
 import org.apache.datasketches.common.SketchesStateException;
-import org.apache.datasketches.memory.DefaultMemoryRequestServer;
 import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.memory.WritableMemory;
 import org.testng.annotations.Test;
@@ -47,7 +46,7 @@ public void checkGrow() {
     int n = 8; //put lgConfigK == 4 into HLL mode
     long bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
     HllSketch hllSketch;
-    WritableMemory wmem = WritableMemory.allocateDirect(bytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmem = WritableMemory.allocateDirect(bytes, Arena.ofConfined());
 
     hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
     for (int i = 0; i < n; i++) {
@@ -56,7 +55,7 @@ public void checkGrow() {
     hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values
     hllSketch.couponUpdate(HllUtil.pair(8, 15));
     hllSketch.couponUpdate(HllUtil.pair(9, 15));
-    //println(hllSketch.toString(true, true, true, true));
+    println(hllSketch.toString(true, true, true, true));
     DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl;
     assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2);
     assertTrue(hllSketch.isMemory());
@@ -74,14 +73,14 @@ public void checkGrow() {
     byteArray = hllSketch.toUpdatableByteArray();
     WritableMemory wmem2 = WritableMemory.writableWrap(byteArray);
     hllSketch2 = HllSketch.writableWrap(wmem2);
-    //println(hllSketch2.toString(true, true, true, true));
+    println(hllSketch2.toString(true, true, true, true));
     DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl;
     assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2);
     assertEquals(dha2.getAuxHashMap().getAuxCount(), 3);
 
     //Check grow to on-heap
     hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow
-    //println(hllSketch.toString(true, true, true, true));
+    println(hllSketch.toString(true, true, true, true));
     dha = (DirectHllArray) hllSketch.hllSketchImpl;
     assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3);
     assertEquals(dha.getAuxHashMap().getAuxCount(), 4);
diff --git a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java
index 1b00e0301..6da2f05fe 100644
--- a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java
+++ b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java
@@ -25,12 +25,11 @@
 import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 
-import org.testng.annotations.Test;
-
-import jdk.incubator.foreign.ResourceScope;
+import java.lang.foreign.Arena;
 
 import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
 
 /**
  * @author Lee Rhodes
@@ -71,7 +70,8 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool
     //println("DIRECT");
     byte[] barr1;
     WritableMemory wmem;
-    try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes)).scope()) {
+    try (Arena arena = Arena.ofConfined()) {
+      wmem = WritableMemory.allocateDirect(bytes, arena);
       hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
       assertTrue(hllSketch.isEmpty());
 
diff --git a/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java
index 9fc74d97b..2e6cc1ea1 100644
--- a/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java
@@ -599,15 +599,18 @@ public void checkReadOnlyExceptions() {
   @Test
   public void checkIsSameResource() {
     int cap = 128;
-    WritableMemory wmem = WritableMemory.allocate(cap);
+    WritableMemory wmem = WritableMemory.allocate(cap); //heap
     WritableMemory reg1 = wmem.writableRegion(0, 64);
     WritableMemory reg2 = wmem.writableRegion(64, 64);
     assertFalse(reg1 == reg2);
-    assertFalse(reg1.isSameResource(reg2));
+    assertFalse(reg1.isSameResource(reg2)); //same original resource, but different offsets
 
     WritableMemory reg3 = wmem.writableRegion(0, 64);
     assertFalse(reg1 == reg3);
-    assertTrue(reg1.isSameResource(reg3));
+    assertTrue(reg1.isSameResource(reg3)); //same original resource, same offsets, different views.
+    reg1.putInt(0, -1);
+    assertEquals(-1, reg3.getInt(0)); //proof
+    reg1.putInt(0, 0);
 
     byte[] byteArr1 = KllItemsSketch.newHeapInstance(20, Comparator.naturalOrder(), serDe).toByteArray();
     reg1.putByteArray(0, byteArr1, 0, byteArr1.length);
@@ -615,11 +618,11 @@ public void checkIsSameResource() {
 
     byte[] byteArr2 = KllItemsSketch.newHeapInstance(20, Comparator.naturalOrder(), serDe).toByteArray();
     reg2.putByteArray(0, byteArr2, 0, byteArr2.length);
-    assertFalse(sk1.isSameResource(reg2));
+    assertFalse(sk1.isSameResource(reg2)); //same original resource, but different offsets
 
     byte[] byteArr3 = KllItemsSketch.newHeapInstance(20, Comparator.naturalOrder(), serDe).toByteArray();
     reg3.putByteArray(0, byteArr3, 0, byteArr3.length);
-    assertTrue(sk1.isSameResource(reg3));
+    assertTrue(sk1.isSameResource(reg3)); //same original resource, same offsets, different views.
   }
 
   // New added tests specially for KllItemsSketch
diff --git a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java
index e51be4b20..2b67f9772 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java
@@ -27,7 +27,8 @@
 
 import org.testng.annotations.Test;
 
-import jdk.incubator.foreign.ResourceScope;
+import java.lang.foreign.Arena;
+import java.nio.ByteOrder;
 
 import org.apache.datasketches.memory.WritableMemory;
 import org.apache.datasketches.quantilescommon.QuantilesDoublesSketchIterator;
@@ -63,8 +64,8 @@ public void test() {
     DoublesSketch.setRandom(1); //make deterministic for test
     DoublesUnion dUnion;
     DoublesSketch dSketch;
-    WritableMemory wmem;
-    try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(10_000_000)).scope()) {
+    try (Arena arena = Arena.ofConfined()) {
+      WritableMemory wmem = WritableMemory.allocateDirect(10_000_000, arena);
       dUnion = DoublesUnion.builder().setMaxK(8).build(wmem);
       for (int s = 0; s < numSketches; s++) { dUnion.union(sketchArr[s]); }
       dSketch = dUnion.getResult(); //result is on heap
diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
index 5195f4187..1c8819981 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
@@ -24,6 +24,7 @@
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 
+import java.lang.foreign.Arena;
 import java.nio.ByteOrder;
 
 import org.apache.datasketches.memory.DefaultMemoryRequestServer;
@@ -47,7 +48,7 @@ public void checkLimitedMemoryScenarios() { //Requesting application
 
     //########## Owning Implementation
     // This part would actually be part of the Memory owning implementation so it is faked here
-    WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmem = WritableMemory.allocateDirect(initBytes, Arena.ofConfined());
     WritableMemory wmem2 = wmem;
     println("Initial mem size: " + wmem.getCapacity());
 
@@ -80,7 +81,7 @@ public void checkGrowBaseBuf() {
     final int u = 32; // don't need the BB to fill here
     final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything
 
-    WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmem = WritableMemory.allocateDirect(initBytes, Arena.ofConfined());
     WritableMemory wmem2 = wmem;
     println("Initial mem size: " + wmem.getCapacity());
     final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
@@ -99,7 +100,7 @@ public void checkGrowCombBuf() {
     final int u = (2 * k) - 1; //just to fill the BB
     final int initBytes = ((2 * k) + 4) << 3; //just room for BB
 
-    WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmem = WritableMemory.allocateDirect(initBytes, Arena.ofConfined());
     WritableMemory wmem2 = wmem;
     println("Initial mem size: " + wmem.getCapacity());
     final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
@@ -115,15 +116,28 @@ public void checkGrowCombBuf() {
     assertFalse(wmem2.isAlive());
   }
 
+  @Test
+  public void checkUpdatableStorageBytes() {
+    final int k = 16;
+    final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, 1);
+    println("Predicted Updatable Storage Bytes: " + initBytes);
+    final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build();
+    usk1.update(1.0);
+    byte[] uarr = usk1.toByteArray();
+    println("Actual Storage Bytes " + uarr.length);
+    assertEquals(initBytes, uarr.length); //64
+  }
+
+
   @Test
   public void checkGrowFromWrappedEmptySketch() {
     final int k = 16;
     final int n = 0;
-    final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, n); //8 bytes
+    final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, n); //empty: 8 bytes
     final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build();
-    final Memory origSketchMem = Memory.wrap(usk1.toByteArray());
+    final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); //on heap
 
-    WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmem = WritableMemory.allocateDirect(initBytes, Arena.ofConfined()); //off heap
     WritableMemory wmem2 = wmem;
     origSketchMem.copyTo(0, wmem, 0, initBytes);
     UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem);
@@ -133,11 +147,11 @@ public void checkGrowFromWrappedEmptySketch() {
     assertTrue(usk2.isEmpty());
 
     //update the sketch forcing it to grow on-heap
-    for (int i = 1; i <= 5; i++) { usk2.update(i); }
-    assertEquals(usk2.getN(), 5);
+    usk2.update(1.0);
+    assertEquals(usk2.getN(), 1);
     WritableMemory mem2 = usk2.getMemory();
     assertFalse(wmem.isSameResource(mem2));
-    assertFalse(mem2.isDirect()); //should now be on-heap
+    assertFalse(mem2.isDirect()); //should now be on-heap //TODO
 
     final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3);
     assertEquals(mem2.getCapacity(), expectedSize);
diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
index 8cdc7bf71..762cfc057 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
@@ -34,7 +34,7 @@
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
-import jdk.incubator.foreign.ResourceScope;
+import java.lang.foreign.Arena;
 
 public class DoublesSketchTest {
 
@@ -141,7 +141,7 @@ public void checkEmptyExceptions() {
 
   @Test
   public void directSketchShouldMoveOntoHeapEventually() {
-    WritableMemory wmem = WritableMemory.allocateDirect(1000, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmem = WritableMemory.allocateDirect(1000, Arena.ofConfined());
     WritableMemory wmem2 = wmem;
     UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
     Assert.assertTrue(sketch.isSameResource(wmem));
@@ -155,7 +155,7 @@ public void directSketchShouldMoveOntoHeapEventually() {
   @Test
   public void directSketchShouldMoveOntoHeapEventually2() {
     int i = 0;
-    WritableMemory wmem = WritableMemory.allocateDirect(50, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmem = WritableMemory.allocateDirect(50, Arena.ofConfined());
     WritableMemory wmem2 = wmem;
     UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
     Assert.assertTrue(sketch.isSameResource(wmem));
@@ -172,10 +172,8 @@ public void directSketchShouldMoveOntoHeapEventually2() {
 
   @Test
   public void checkEmptyDirect() {
-    WritableMemory wmem ;
-    try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(1000, 1,
-            ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) {
-
+    try (Arena arena = Arena.ofConfined()) {
+       WritableMemory wmem = WritableMemory.allocateDirect(1000, Arena.ofConfined());
       UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
       sketch.toByteArray(); //exercises a specific path
     } catch (final Exception e) {
diff --git a/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java b/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java
index adf916ef3..765d4d037 100644
--- a/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java
@@ -37,24 +37,19 @@
 import static org.apache.datasketches.quantiles.PreambleUtil.insertSerVer;
 import static org.testng.Assert.assertEquals;
 
-import java.nio.ByteOrder;
+import java.lang.foreign.Arena;
 
-import org.apache.datasketches.memory.DefaultMemoryRequestServer;
 import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.memory.WritableMemory;
 import org.testng.annotations.Test;
 
-import jdk.incubator.foreign.ResourceScope;
-
 public class PreambleUtilTest {
 
   @Test
   public void checkInsertsAndExtracts() {
     final int bytes = 32;
-    WritableMemory offHeapMem;
-    try (ResourceScope scope = (offHeapMem = WritableMemory.allocateDirect(bytes, 1,
-            ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) {
-
+    try (Arena arena = Arena.ofConfined()) {
+        WritableMemory offHeapMem = WritableMemory.allocateDirect(bytes, arena);
       final WritableMemory onHeapMem = WritableMemory.writableWrap(new byte[bytes]);
 
       onHeapMem.clear();
diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
index e0f79087b..cc5be6a85 100644
--- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
@@ -32,7 +32,7 @@
 import org.apache.datasketches.memory.WritableMemory;
 import org.testng.annotations.Test;
 
-import jdk.incubator.foreign.ResourceScope;
+import java.lang.foreign.Arena;
 
 /**
  * @author Lee Rhodes
@@ -79,8 +79,8 @@ public void checkHeapifyWrap(int k, int u, boolean ordered) {
     //Prepare Memory for direct
     int bytes = usk.getCompactBytes(); //for Compact
 
-    WritableMemory directMem;
-    try (ResourceScope scope = (directMem = WritableMemory.allocateDirect(bytes)).scope()) {
+    try (Arena arena = Arena.ofConfined()) {
+      WritableMemory directMem = WritableMemory.allocateDirect(bytes, arena);
 
       /**Via CompactSketch.compact**/
       refSk = usk.compact(ordered, directMem);
diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
index f36597b7c..5f608113a 100644
--- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
@@ -38,6 +38,7 @@
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 
+import java.lang.foreign.Arena;
 import java.nio.ByteOrder;
 import java.util.Arrays;
 
@@ -52,8 +53,6 @@
 import org.apache.datasketches.thetacommon.ThetaUtil;
 import org.testng.annotations.Test;
 
-import jdk.incubator.foreign.ResourceScope;
-
 /**
  * @author Lee Rhodes
  */
@@ -62,8 +61,8 @@ public class DirectQuickSelectSketchTest {
   @Test//(expectedExceptions = SketchesArgumentException.class)
   public void checkBadSerVer() {
     int k = 512;
-    WritableMemory wmem;
-    try (ResourceScope scope = (wmem = makeNativeMemory(k)).scope()) {
+    try (Arena arena = Arena.ofConfined()) {
+        WritableMemory wmem = makeNativeMemory(k, arena);
 
       UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(wmem);
       DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks
@@ -88,8 +87,8 @@ public void checkBadSerVer() {
   @Test
   public void checkConstructorKtooSmall() {
     int k = 8;
-    WritableMemory wmem;
-    try (ResourceScope scope = (wmem = makeNativeMemory(k)).scope()) {
+    try (Arena arena = Arena.ofConfined()) {
+        WritableMemory wmem = makeNativeMemory(k, arena);
       UpdateSketch.builder().setNominalEntries(k).build(wmem);
     } catch (final Exception e) {
       if (e instanceof SketchesArgumentException) {}
@@ -100,8 +99,8 @@ public void checkConstructorKtooSmall() {
   @Test
   public void checkConstructorMemTooSmall() {
     int k = 16;
-    WritableMemory wmem;
-    try (ResourceScope scope = (wmem = makeNativeMemory(k/2)).scope()) {
+    try (Arena arena = Arena.ofConfined()) {
+        WritableMemory wmem = makeNativeMemory(k/2, arena);
       UpdateSketch.builder().setNominalEntries(k).build(wmem);
     } catch (final Exception e) {
       if (e instanceof SketchesArgumentException) {}
@@ -126,10 +125,8 @@ public void checkHeapifyIllegalFamilyID_heapify() {
   public void checkHeapifyMemoryEstimating() {
     int k = 512;
     int u = 2*k; //thus estimating
-
-    WritableMemory wmem;
-    try (ResourceScope scope = (wmem = makeNativeMemory(k)).scope()) {
-
+    try (Arena arena = Arena.ofConfined()) {
+      WritableMemory wmem = makeNativeMemory(k, arena);
       UpdateSketch sk1 = UpdateSketch.builder().setNominalEntries(k).build(wmem);
       for (int i=0; i
Date: Thu, 20 Feb 2025 15:59:19 -0800
Subject: [PATCH 71/88] Update .asf.yaml:

---
 .asf.yaml | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/.asf.yaml b/.asf.yaml
index 50695edc9..5628899f8 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -4,15 +4,24 @@ github:
   ghp_branch: gh-pages
   ghp_path: /docs
 
-  master:
-    required_status_checks:
-      # strict means "Require branches to be up to date before merging."
-      strict: true
-
-    required_pull_request_reviews:
-      dismiss_stale_reviews: true
-      required_approving_review_count: 1
+  protected_branches:
+    main:
+      required_status_checks:
+        # strict means "Require branches to be up to date before merging."
+        strict: true
 
+      required_pull_request_reviews:
+        dismiss_stale_reviews: true
+        equired_approving_review_count: 1
+      
+      # squash or rebase must be allowed in the repo for this setting to be set to true.
+      required_linear_history: false
+      
+      required_signatures: false
+      
+      # requires all conversations to be resolved before merging is possible
+      required_conversation_resolution: false
+      
   dependabot_alerts:  true
   dependabot_updates: false
 

From bfb9f96823256a865d3268e5b542f051e49a4b91 Mon Sep 17 00:00:00 2001
From: Lee Rhodes 
Date: Thu, 20 Feb 2025 16:13:19 -0800
Subject: [PATCH 72/88] change dismiss state reviews to false.

---
 .asf.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.asf.yaml b/.asf.yaml
index 5628899f8..f7627bc08 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -11,7 +11,7 @@ github:
         strict: true
 
       required_pull_request_reviews:
-        dismiss_stale_reviews: true
+        dismiss_stale_reviews: false
         equired_approving_review_count: 1
       
       # squash or rebase must be allowed in the repo for this setting to be set to true.

From 42a8a2686f8e86087862a813b7600cf81310095f Mon Sep 17 00:00:00 2001
From: jmalkin <786705+jmalkin@users.noreply.github.com>
Date: Fri, 21 Feb 2025 10:33:21 -0800
Subject: [PATCH 73/88] Fix a few typos in BloomFilter javadocs

---
 .../filters/bloomfilter/BloomFilter.java      | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java
index 171fc2cfb..7c166a29d 100644
--- a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java
+++ b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java
@@ -381,7 +381,7 @@ private void updateInternal(final long h0, final long h1) {
 
   /**
    * Updates the filter with the provided long and
-   * returns the result from quering that value prior to the update.
+   * returns the result from querying that value prior to the update.
    * @param item an item with which to update the filter
    * @return The query result prior to applying the update
    */
@@ -393,7 +393,7 @@ public boolean queryAndUpdate(final long item) {
 
   /**
    * Updates the filter with the provided double and
-   * returns the result from quering that value prior to the update.
+   * returns the result from querying that value prior to the update.
    * The double is canonicalized (NaN and +/- infinity) in the call.
    * @param item an item with which to update the filter
    * @return The query result prior to applying the update
@@ -408,7 +408,7 @@ public boolean queryAndUpdate(final double item) {
 
   /**
    * Updates the filter with the provided String and
-   * returns the result from quering that value prior to the update.
+   * returns the result from querying that value prior to the update.
    * The string is converted to a byte array using UTF8 encoding.
    *
    * 
Note: this will not produce the same output hash values as the {@link #queryAndUpdate(char[])}
@@ -428,7 +428,7 @@ public boolean queryAndUpdate(final String item) {
 
   /**
    * Updates the filter with the provided byte[] and
-   * returns the result from quering that array prior to the update.
+   * returns the result from querying that array prior to the update.
    * @param data an array with which to update the filter
    * @return The query result prior to applying the update, or false if data is null
    */
@@ -440,7 +440,7 @@ public boolean queryAndUpdate(final byte[] data) {
 
   /**
    * Updates the filter with the provided char[] and
-   * returns the result from quering that array prior to the update.
+   * returns the result from querying that array prior to the update.
    * @param data an array with which to update the filter
    * @return The query result prior to applying the update, or false if data is null
    */
@@ -453,7 +453,7 @@ public boolean queryAndUpdate(final char[] data) {
 
   /**
    * Updates the filter with the provided short[] and
-   * returns the result from quering that array prior to the update.
+   * returns the result from querying that array prior to the update.
    * @param data an array with which to update the filter
    * @return The query result prior to applying the update, or false if data is null
    */
@@ -466,7 +466,7 @@ public boolean queryAndUpdate(final short[] data) {
 
   /**
    * Updates the filter with the provided int[] and
-   * returns the result from quering that array prior to the update.
+   * returns the result from querying that array prior to the update.
    * @param data an array with which to update the filter
    * @return The query result prior to applying the update, or false if data is null
    */
@@ -479,7 +479,7 @@ public boolean queryAndUpdate(final int[] data) {
 
   /**
    * Updates the filter with the provided long[] and
-   * returns the result from quering that array prior to the update.
+   * returns the result from querying that array prior to the update.
    * @param data an array with which to update the filter
    * @return The query result prior to applying the update, or false if data is null
    */
@@ -492,7 +492,7 @@ public boolean queryAndUpdate(final long[] data) {
 
   /**
    * Updates the filter with the provided Memory and
-   * returns the result from quering that Memory prior to the update.
+   * returns the result from querying that Memory prior to the update.
    * @param mem an array with which to update the filter
    * @return The query result prior to applying the update, or false if mem is null
    */
@@ -762,7 +762,7 @@ public static long getSerializedSize(final long numBits) {
  *  3   ||---------------------------------NumBitsSet------------------------------------|
  *