Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -551,7 +551,7 @@ public void testReader() throws IOException
Assert.assertEquals("world", nextRow.getDimension("dim2").get(0));
}

Assert.assertEquals(2 * CONTENT.length, inputStats.getProcessedBytes());
Assert.assertEquals(2L * CONTENT.length, inputStats.getProcessedBytes());
EasyMock.verify(OSSCLIENT);
}

Expand Down Expand Up @@ -598,7 +598,7 @@ public void testCompressedReader() throws IOException
Assert.assertEquals("world", nextRow.getDimension("dim2").get(0));
}

Assert.assertEquals(2 * CONTENT.length, inputStats.getProcessedBytes());
Assert.assertEquals(2L * CONTENT.length, inputStats.getProcessedBytes());
EasyMock.verify(OSSCLIENT);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,8 @@ public void test_taskLog_fetch_withNegativeRange() throws IOException
EasyMock.replay(ossClient);

OssTaskLogs ossTaskLogs = getOssTaskLogs();
Optional<InputStream> inputStreamOptional = ossTaskLogs.streamTaskLog(KEY_1, -1 * (LOG_CONTENTS.length() - 1));
final Optional<InputStream> inputStreamOptional =
ossTaskLogs.streamTaskLog(KEY_1, 1L - LOG_CONTENTS.length());
String taskLogs = new BufferedReader(
new InputStreamReader(inputStreamOptional.get(), StandardCharsets.UTF_8))
.lines()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,9 @@ public int getArraySize()
* @return the entry
*/
@Override
public int getArrayEntry(int idx)
public int getArrayEntry(final int idx)
{
return buf.getInt(position + idx * Integer.BYTES);
return buf.getInt(bufferPosition(idx));
}

/**
Expand All @@ -131,9 +131,15 @@ public int getArrayEntry(int idx)
* @param val value to set
*/
@Override
protected void setArrayEntry(int idx, int val)
protected void setArrayEntry(final int idx, final int val)
{
buf.putInt(position + idx * Integer.BYTES, val);
buf.putInt(bufferPosition(idx), val);
}

private int bufferPosition(final int idx)
{
Preconditions.checkElementIndex(idx, size);
return Math.addExact(position, Math.multiplyExact(idx, Integer.BYTES));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,9 @@ protected CompressedBigDecimalAggregatorFactoryBase(
this.scale = scale == null ? DEFAULT_SCALE : scale;
this.strictNumberParsing = strictNumberParsing == null ? DEFAULT_STRICT_NUMBER_PARSING : strictNumberParsing;

byte[] fieldNameBytes = StringUtils.toUtf8(fieldName);
final byte[] fieldNameBytes = StringUtils.toUtf8(fieldName);

cacheKey = ByteBuffer.allocate(1 + fieldNameBytes.length + 2 * Integer.BYTES + 1)
cacheKey = ByteBuffer.allocate(Math.addExact(fieldNameBytes.length, 2 * Integer.BYTES + 2))
.put(cacheTypeId)
.put(fieldNameBytes)
.putInt(this.size)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.datadoghq.sketch.ddsketch.DDSketch;
import com.datadoghq.sketch.ddsketch.DDSketchProtoBinding;
import com.datadoghq.sketch.ddsketch.store.CollapsingLowestDenseStore;
import com.google.common.base.Preconditions;
import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.druid.segment.data.ObjectStrategy;

Expand All @@ -44,11 +45,18 @@ public DDSketch fromByteBuffer(ByteBuffer buffer, int numBytes)
if (numBytes == 0) {
return null;
}
ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer();
readOnlyBuffer.limit(buffer.position() + numBytes);
Preconditions.checkArgument(
numBytes >= 0 && numBytes <= buffer.remaining(),
"numBytes[%s] exceeds buffer remaining[%s]",
numBytes,
buffer.remaining()
);
final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer();
readOnlyBuffer.limit(Math.addExact(buffer.position(), numBytes));
try {
com.datadoghq.sketch.ddsketch.proto.DDSketch proto = com.datadoghq.sketch.ddsketch.proto.DDSketch.parseFrom(readOnlyBuffer);
DDSketch recovered = DDSketchProtoBinding.fromProto(() -> new CollapsingLowestDenseStore(1000), proto);
final com.datadoghq.sketch.ddsketch.proto.DDSketch proto =
com.datadoghq.sketch.ddsketch.proto.DDSketch.parseFrom(readOnlyBuffer);
final DDSketch recovered = DDSketchProtoBinding.fromProto(() -> new CollapsingLowestDenseStore(1000), proto);
return recovered;
}
catch (InvalidProtocolBufferException e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.druid.query.aggregation.momentsketch;

import com.google.common.base.Preconditions;
import org.apache.druid.query.aggregation.momentsketch.aggregator.MomentSketchAggregatorFactory;
import org.apache.druid.segment.data.ObjectStrategy;

Expand All @@ -41,7 +42,13 @@ public MomentSketchWrapper fromByteBuffer(ByteBuffer buffer, int numBytes)
if (numBytes == 0) {
return null;
}
buffer.limit(buffer.position() + numBytes);
Preconditions.checkArgument(
numBytes > 0 && numBytes <= buffer.remaining(),
"numBytes[%s] exceeds buffer remaining[%s]",
numBytes,
buffer.remaining()
);
buffer.limit(Math.addExact(buffer.position(), numBytes));
return MomentSketchWrapper.fromBytes(buffer);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public void testToBytesSmallValues()
byte[] bytes = histogram.toBytes();
int keySize = Short.BYTES;
int valSize = 0;
Assert.assertEquals("Should compact small values within key bytes", 5 * (keySize + valSize), bytes.length);
Assert.assertEquals("Should compact small values within key bytes", 5L * (keySize + valSize), bytes.length);

SpectatorHistogram deserialized = SpectatorHistogram.deserialize(bytes);
Assert.assertEquals(1L, deserialized.get(PercentileBuckets.indexOf(10)));
Expand Down Expand Up @@ -86,7 +86,7 @@ public void testToBytesSmallishValues()
byte[] bytes = histogram.toBytes();
int keySize = Short.BYTES;
int valSize = Byte.BYTES;
Assert.assertEquals("Should compact small values to a byte", 5 * (keySize + valSize), bytes.length);
Assert.assertEquals("Should compact small values to a byte", 5L * (keySize + valSize), bytes.length);

SpectatorHistogram deserialized = SpectatorHistogram.deserialize(bytes);
Assert.assertEquals(64L, deserialized.get(PercentileBuckets.indexOf(10)));
Expand Down Expand Up @@ -115,7 +115,7 @@ public void testToBytesMedValues()
byte[] bytes = histogram.toBytes();
int keySize = Short.BYTES;
int valSize = Short.BYTES;
Assert.assertEquals("Should compact medium values to short", 5 * (keySize + valSize), bytes.length);
Assert.assertEquals("Should compact medium values to short", 5L * (keySize + valSize), bytes.length);

SpectatorHistogram deserialized = SpectatorHistogram.deserialize(bytes);
Assert.assertEquals(512L, deserialized.get(PercentileBuckets.indexOf(10)));
Expand Down Expand Up @@ -144,7 +144,7 @@ public void testToBytesLargerValues()
byte[] bytes = histogram.toBytes();
int keySize = Short.BYTES;
int valSize = Integer.BYTES;
Assert.assertEquals("Should compact larger values to integer", 5 * (keySize + valSize), bytes.length);
Assert.assertEquals("Should compact larger values to integer", 5L * (keySize + valSize), bytes.length);

SpectatorHistogram deserialized = SpectatorHistogram.deserialize(bytes);
Assert.assertEquals(100000L, deserialized.get(PercentileBuckets.indexOf(10)));
Expand Down Expand Up @@ -173,7 +173,7 @@ public void testToBytesBiggestValues()
byte[] bytes = histogram.toBytes();
int keySize = Short.BYTES;
int valSize = Long.BYTES;
Assert.assertEquals("Should not compact larger values", 5 * (keySize + valSize), bytes.length);
Assert.assertEquals("Should not compact larger values", 5L * (keySize + valSize), bytes.length);

SpectatorHistogram deserialized = SpectatorHistogram.deserialize(bytes);
Assert.assertEquals(10000000000L, deserialized.get(PercentileBuckets.indexOf(10)));
Expand Down Expand Up @@ -201,7 +201,7 @@ public void testToBytesMixedValues()

byte[] bytes = histogram.toBytes();
int keySize = Short.BYTES;
Assert.assertEquals("Should not compact larger values", (5 * keySize) + 0 + 2 + 4 + 8 + 8, bytes.length);
Assert.assertEquals("Should not compact larger values", (5L * keySize) + 0 + 2 + 4 + 8 + 8, bytes.length);

SpectatorHistogram deserialized = SpectatorHistogram.deserialize(bytes);
Assert.assertEquals(1L, deserialized.get(PercentileBuckets.indexOf(10)));
Expand Down Expand Up @@ -232,7 +232,7 @@ public void testToBytesBoundaryValues()

byte[] bytes = histogram.toBytes();
int keySize = Short.BYTES;
Assert.assertEquals("Should compact", (8 * keySize) + 0 + 1 + 1 + 2 + 2 + 4 + 4 + 8, bytes.length);
Assert.assertEquals("Should compact", (8L * keySize) + 0 + 1 + 1 + 2 + 2 + 4 + 4 + 8, bytes.length);

SpectatorHistogram deserialized = SpectatorHistogram.deserialize(bytes);
Assert.assertEquals(63L, deserialized.get(6));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.druid.query.aggregation.tdigestsketch;

import com.google.common.base.Preconditions;
import com.tdunning.math.stats.MergingDigest;
import org.apache.druid.segment.data.ObjectStrategy;

Expand All @@ -41,8 +42,14 @@ public MergingDigest fromByteBuffer(ByteBuffer buffer, int numBytes)
if (numBytes == 0) {
return null;
}
ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer();
readOnlyBuffer.limit(buffer.position() + numBytes);
Preconditions.checkArgument(
numBytes > 0 && numBytes <= buffer.remaining(),
"numBytes[%s] exceeds buffer remaining[%s]",
numBytes,
buffer.remaining()
);
final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer();
readOnlyBuffer.limit(Math.addExact(buffer.position(), numBytes));
return MergingDigest.fromBytes(readOnlyBuffer);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,7 @@ public void testReader() throws IOException
Assert.assertEquals("hello", nextRow.getDimension("dim1").get(0));
Assert.assertEquals("world", nextRow.getDimension("dim2").get(0));
}
Assert.assertEquals(2 * CONTENT.length, inputStats.getProcessedBytes());
Assert.assertEquals(2L * CONTENT.length, inputStats.getProcessedBytes());
}

@Test
Expand Down Expand Up @@ -466,7 +466,7 @@ public void testCompressedReader() throws IOException
Assert.assertEquals("hello", nextRow.getDimension("dim1").get(0));
Assert.assertEquals("world", nextRow.getDimension("dim2").get(0));
}
Assert.assertEquals(2 * CONTENT.length, inputStats.getProcessedBytes());
Assert.assertEquals(2L * CONTENT.length, inputStats.getProcessedBytes());
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1261,7 +1261,7 @@ public void toBytesCompact(ByteBuffer buf)
for (int i = 0; i < binCount; ++i) {
// repeat each value bins[i] times for approximate bins
if ((bins[i] & APPROX_FLAG_BIT) != 0) {
for (int k = 0; k < (bins[i] & COUNT_BITS); ++k) {
for (long k = 0; k < (bins[i] & COUNT_BITS); ++k) {
buf.putFloat(positions[i]);
}
}
Expand All @@ -1277,7 +1277,7 @@ public void toBytesCompact(ByteBuffer buf)
for (int i = 0; i < binCount; ++i) {
// repeat each value bins[i] times for exact bins
if ((bins[i] & APPROX_FLAG_BIT) == 0) {
for (int k = 0; k < (bins[i] & COUNT_BITS); ++k) {
for (long k = 0; k < (bins[i] & COUNT_BITS); ++k) {
buf.putFloat(positions[i]);
}
}
Expand Down Expand Up @@ -1371,7 +1371,7 @@ public static ApproximateHistogram fromBytesCompact(ByteBuffer buf)
}
return histogram;
} else {
byte approxCount = (byte) (-1 * count);
final int approxCount = -count;

Map<Float, Long> approx = new HashMap<>();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.druid.query.aggregation.histogram;

import com.google.common.base.Preconditions;
import it.unimi.dsi.fastutil.bytes.ByteArrays;
import org.apache.druid.data.input.InputRow;
import org.apache.druid.data.input.Rows;
Expand Down Expand Up @@ -93,7 +94,13 @@ public Class<ApproximateHistogram> getClazz()
@Override
public ApproximateHistogram fromByteBuffer(ByteBuffer buffer, int numBytes)
{
buffer.limit(buffer.position() + numBytes);
Preconditions.checkArgument(
numBytes >= 0 && numBytes <= buffer.remaining(),
"numBytes[%s] exceeds buffer remaining[%s]",
numBytes,
buffer.remaining()
);
buffer.limit(Math.addExact(buffer.position(), numBytes));
return ApproximateHistogram.fromBytes(buffer);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -993,7 +993,7 @@ protected static FixedBucketsHistogram fromByteBufferFullNoSerdeHeader(ByteBuffe
double lowerLimit = buf.getDouble();
double upperLimit = buf.getDouble();
int numBuckets = buf.getInt();
OutlierHandlingMode outlierHandlingMode = OutlierHandlingMode.values()[buf.get()];
final OutlierHandlingMode outlierHandlingMode = readOutlierHandlingMode(buf);

long count = buf.getLong();
long lowerOutlierCount = buf.getLong();
Expand Down Expand Up @@ -1034,7 +1034,7 @@ private static FixedBucketsHistogram fromBytesSparse(ByteBuffer buf)
double lowerLimit = buf.getDouble();
double upperLimit = buf.getDouble();
int numBuckets = buf.getInt();
OutlierHandlingMode outlierHandlingMode = OutlierHandlingMode.values()[buf.get()];
final OutlierHandlingMode outlierHandlingMode = readOutlierHandlingMode(buf);

long count = buf.getLong();
long lowerOutlierCount = buf.getLong();
Expand Down Expand Up @@ -1067,6 +1067,21 @@ private static FixedBucketsHistogram fromBytesSparse(ByteBuffer buf)
);
}

private static OutlierHandlingMode readOutlierHandlingMode(final ByteBuffer buf)
{
final byte mode = buf.get();
switch (mode) {
case 0:
return OutlierHandlingMode.IGNORE;
case 1:
return OutlierHandlingMode.OVERFLOW;
case 2:
return OutlierHandlingMode.CLIP;
default:
throw new ISE("Invalid outlier handling mode [%s].", mode);
}
}

/**
* Compute the size in bytes of a full-encoding serialized histogram, without the serialization header
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.druid.query.aggregation.histogram;

import com.google.common.base.Preconditions;
import com.google.common.collect.Ordering;
import org.apache.druid.data.input.InputRow;
import org.apache.druid.data.input.Rows;
Expand Down Expand Up @@ -134,8 +135,14 @@ public Class<? extends FixedBucketsHistogram> getClazz()
@Override
public FixedBucketsHistogram fromByteBuffer(ByteBuffer buffer, int numBytes)
{
buffer.limit(buffer.position() + numBytes);
FixedBucketsHistogram fbh = FixedBucketsHistogram.fromByteBuffer(buffer);
Preconditions.checkArgument(
numBytes >= 0 && numBytes <= buffer.remaining(),
"numBytes[%s] exceeds buffer remaining[%s]",
numBytes,
buffer.remaining()
);
buffer.limit(Math.addExact(buffer.position(), numBytes));
final FixedBucketsHistogram fbh = FixedBucketsHistogram.fromByteBuffer(buffer);
return fbh;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public class MapPopulatorTest
public void test_getByteLengthOfObject_string_stringLength()
{
String o = "string";
Assert.assertEquals((o.length() * Character.BYTES) + 40, MapPopulator.getByteLengthOfObject(o));
Assert.assertEquals(((long) o.length() * Character.BYTES) + 40, MapPopulator.getByteLengthOfObject(o));
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,8 @@ public void test_taskLog_fetch_withNegativeRange() throws IOException

S3TaskLogs s3TaskLogs = getS3TaskLogs();

Optional<InputStream> inputStreamOptional = s3TaskLogs.streamTaskLog(KEY_1, -1 * (LOG_CONTENTS.length() - 1));
final Optional<InputStream> inputStreamOptional =
s3TaskLogs.streamTaskLog(KEY_1, 1L - LOG_CONTENTS.length());
String taskLogs = new BufferedReader(
new InputStreamReader(inputStreamOptional.get(), StandardCharsets.UTF_8))
.lines()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ public void testWriteAndHappy() throws IOException
}
// each chunk is 10 bytes, so there should be 10 chunks.
Assert.assertEquals(10, s3.partRequests.size());
s3.assertCompleted(chunkSize, Integer.BYTES * 25);
s3.assertCompleted(chunkSize, Integer.BYTES * 25L);
}

@Test
Expand All @@ -144,7 +144,7 @@ public void testWriteSizeLargerThanConfiguredMaxChunkSizeShouldSucceed() throws
}
// each chunk 10 bytes, so there should be 2 chunks.
Assert.assertEquals(2, s3.partRequests.size());
s3.assertCompleted(chunkSize, Integer.BYTES * 3);
s3.assertCompleted(chunkSize, Integer.BYTES * 3L);
}

@Test
Expand Down Expand Up @@ -195,7 +195,7 @@ public void testSuccessToUploadAfterRetry() throws IOException
}
// each chunk is 10 bytes, so there should be 10 chunks.
Assert.assertEquals(10, s3.partRequests.size());
s3.assertCompleted(chunkSize, Integer.BYTES * 25);
s3.assertCompleted(chunkSize, Integer.BYTES * 25L);
}

@Test
Expand Down
Loading
Loading