diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/query/QueryTimeoutRuntimeException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/query/QueryTimeoutRuntimeException.java index dec3081ec7a5..ddf4ee9a28c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/query/QueryTimeoutRuntimeException.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/query/QueryTimeoutRuntimeException.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.exception.query; +import com.google.common.math.LongMath; + /** This class is used to throw run time exception when query is time out. */ public class QueryTimeoutRuntimeException extends RuntimeException { public static final String QUERY_TIMEOUT_EXCEPTION_MESSAGE = @@ -27,7 +29,10 @@ public class QueryTimeoutRuntimeException extends RuntimeException { public QueryTimeoutRuntimeException(long startTime, long currentTime, long timeout) { super( String.format( - QUERY_TIMEOUT_EXCEPTION_MESSAGE, startTime, startTime + timeout, currentTime)); + QUERY_TIMEOUT_EXCEPTION_MESSAGE, + startTime, + LongMath.saturatedAdd(startTime, timeout), + currentTime)); } public QueryTimeoutRuntimeException(String message) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessor.java index cfaca4606239..66d5f9c92aaa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessor.java @@ -30,11 +30,13 @@ import org.apache.tsfile.utils.Pair; +import java.math.BigInteger; import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import static com.google.common.math.LongMath.saturatedAdd; import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_SLIDING_BOUNDARY_TIME_DEFAULT_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_SLIDING_BOUNDARY_TIME_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_SLIDING_SECONDS_DEFAULT_VALUE; @@ -78,10 +80,12 @@ public Set mayAddWindow( ? slidingBoundaryTime : windowList.get(windowList.size() - 1).getTimestamp(); - if (timeStamp >= (windowList.isEmpty() ? lastTime : lastTime + slidingInterval)) { + if (windowList.isEmpty() + ? timeStamp >= lastTime + : isTimestampAtOrAfterWindowEnd(timeStamp, lastTime, slidingInterval)) { final TimeSeriesWindow window = new TimeSeriesWindow(this, null); // Align to the last time + k * slidingInterval, k is a natural number - window.setTimestamp(((timeStamp - lastTime) / slidingInterval) * slidingInterval + lastTime); + window.setTimestamp(alignWindowStart(timeStamp, lastTime, slidingInterval)); windowList.add(window); return Collections.singleton(window); } @@ -94,12 +98,12 @@ public Pair updateAndMaySetWindowState( if (timeStamp < window.getTimestamp()) { return new Pair<>(WindowState.IGNORE_VALUE, null); } - if (timeStamp >= window.getTimestamp() + slidingInterval) { + if (isTimestampAtOrAfterWindowEnd(timeStamp, window.getTimestamp(), slidingInterval)) { return new Pair<>( WindowState.EMIT_AND_PURGE_WITHOUT_COMPUTE, new WindowOutput() .setTimestamp(window.getTimestamp()) - .setProgressTime(window.getTimestamp() + slidingInterval)); + .setProgressTime(saturatedAdd(window.getTimestamp(), slidingInterval))); } return new Pair<>(WindowState.COMPUTE, null); } @@ -108,6 +112,23 @@ public Pair updateAndMaySetWindowState( public WindowOutput forceOutput(final TimeSeriesWindow window) { return new WindowOutput() .setTimestamp(window.getTimestamp()) - .setProgressTime(window.getTimestamp() + slidingInterval); + .setProgressTime(saturatedAdd(window.getTimestamp(), slidingInterval)); + } + + private static boolean isTimestampAtOrAfterWindowEnd( + final long timestamp, final long windowStart, final long interval) { + return windowStart <= Long.MAX_VALUE - interval && timestamp >= windowStart + interval; + } + + private static long alignWindowStart( + final long timestamp, final long baseTime, final long interval) { + final BigInteger base = BigInteger.valueOf(baseTime); + final BigInteger intervalValue = BigInteger.valueOf(interval); + return base.add( + BigInteger.valueOf(timestamp) + .subtract(base) + .divide(intervalValue) + .multiply(intervalValue)) + .longValueExact(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/DownSamplingTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/DownSamplingTimeUtils.java new file mode 100644 index 000000000000..b0fee286d853 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/DownSamplingTimeUtils.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.processor.downsampling; + +public class DownSamplingTimeUtils { + + private DownSamplingTimeUtils() { + // Utility class. + } + + public static boolean isTimeDistanceLessThanOrEqualTo(long left, long right, long distance) { + if (distance < 0) { + return false; + } + final long difference = left >= right ? left - right : right - left; + return Long.compareUnsigned(difference, distance) <= 0; + } + + public static boolean isTimeDistanceGreaterThanOrEqualTo(long left, long right, long distance) { + if (distance < 0) { + return true; + } + final long difference = left >= right ? left - right : right - left; + return Long.compareUnsigned(difference, distance) >= 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java index 7dc8c87c09b2..81fbff2bfe90 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java @@ -24,6 +24,9 @@ import java.time.LocalDate; import java.util.Objects; +import static org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceGreaterThanOrEqualTo; +import static org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceLessThanOrEqualTo; + public class ChangingValueFilter { private final ChangingValueSamplingProcessor processor; @@ -59,13 +62,13 @@ public boolean filter(final long timestamp, final T value) { } private boolean tryFilter(final long timestamp, final T value) { - final long timeDiff = Math.abs(timestamp - lastStoredTimestamp); - - if (timeDiff <= processor.getCompressionMinTimeInterval()) { + if (isTimeDistanceLessThanOrEqualTo( + timestamp, lastStoredTimestamp, processor.getCompressionMinTimeInterval())) { return false; } - if (timeDiff >= processor.getCompressionMaxTimeInterval()) { + if (isTimeDistanceGreaterThanOrEqualTo( + timestamp, lastStoredTimestamp, processor.getCompressionMaxTimeInterval())) { reset(timestamp, value); return true; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java index 87a45fd5fa9e..41738e457535 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java @@ -24,6 +24,10 @@ import java.time.LocalDate; import java.util.Objects; +import static org.apache.iotdb.commons.utils.CommonDateTimeUtils.timeDifferenceAsDouble; +import static org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceGreaterThanOrEqualTo; +import static org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceLessThanOrEqualTo; + public class SwingingDoorTrendingFilter { private final SwingingDoorTrendingSamplingProcessor processor; @@ -85,14 +89,13 @@ public boolean filter(final long timestamp, final T value) { } private boolean tryFilter(final long timestamp, final T value) { - final long timeDiff = timestamp - lastStoredTimestamp; - final long absTimeDiff = Math.abs(timeDiff); - - if (absTimeDiff <= processor.getCompressionMinTimeInterval()) { + if (isTimeDistanceLessThanOrEqualTo( + timestamp, lastStoredTimestamp, processor.getCompressionMinTimeInterval())) { return false; } - if (absTimeDiff >= processor.getCompressionMaxTimeInterval()) { + if (isTimeDistanceGreaterThanOrEqualTo( + timestamp, lastStoredTimestamp, processor.getCompressionMaxTimeInterval())) { reset(timestamp, value); return true; } @@ -114,6 +117,7 @@ private boolean tryFilter(final long timestamp, final T value) { final double doubleValue = Double.parseDouble(value.toString()); final double lastStoredDoubleValue = Double.parseDouble(lastStoredValue.toString()); final double valueDiff = doubleValue - lastStoredDoubleValue; + final double timeDiff = timeDifferenceAsDouble(timestamp, lastStoredTimestamp); final double currentUpperSlope = (valueDiff - processor.getCompressionDeviation()) / timeDiff; if (currentUpperSlope > upperDoor) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/tumbling/TumblingTimeSamplingProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/tumbling/TumblingTimeSamplingProcessor.java index 665f5781801a..bbc39924108e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/tumbling/TumblingTimeSamplingProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/tumbling/TumblingTimeSamplingProcessor.java @@ -39,6 +39,7 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_DOWN_SAMPLING_SPLIT_FILE_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_TUMBLING_TIME_INTERVAL_SECONDS_DEFAULT_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_TUMBLING_TIME_INTERVAL_SECONDS_KEY; +import static org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceGreaterThanOrEqualTo; public class TumblingTimeSamplingProcessor extends DownSamplingProcessor { @@ -113,7 +114,8 @@ protected void processRow( final Long lastSampleTime = pathLastObjectCache.getPartialPathLastObject(timeSeriesSuffix); if (lastSampleTime == null - || Math.abs(currentRowTime - lastSampleTime) >= intervalInCurrentPrecision) { + || isTimeDistanceGreaterThanOrEqualTo( + currentRowTime, lastSampleTime, intervalInCurrentPrecision)) { try { rowCollector.collectRow(row); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjector.java index 625c6156a2b3..09f077cc5756 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjector.java @@ -24,6 +24,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.google.common.math.LongMath.saturatedAdd; + public class DataRegionWatermarkInjector { private static final Logger LOGGER = LoggerFactory.getLogger(DataRegionWatermarkInjector.class); @@ -66,7 +68,11 @@ public PipeWatermarkEvent inject() { } private static long calculateNextInjectionTime(long injectionIntervalInMs) { - final long currentTime = System.currentTimeMillis(); - return currentTime / injectionIntervalInMs * injectionIntervalInMs + injectionIntervalInMs; + return calculateNextInjectionTime(System.currentTimeMillis(), injectionIntervalInMs); + } + + static long calculateNextInjectionTime(long currentTime, long injectionIntervalInMs) { + return saturatedAdd( + currentTime / injectionIntervalInMs * injectionIntervalInMs, injectionIntervalInMs); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java index f72a4f06cfbe..90be47c7ab64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java @@ -231,11 +231,11 @@ public void customize( } startTimePartitionIdLowerBound = - (realtimeDataExtractionStartTime % TimePartitionUtils.getTimePartitionInterval() == 0) + TimePartitionUtils.isTimePartitionStartTime(realtimeDataExtractionStartTime) ? TimePartitionUtils.getTimePartitionId(realtimeDataExtractionStartTime) : TimePartitionUtils.getTimePartitionId(realtimeDataExtractionStartTime) + 1; endTimePartitionIdUpperBound = - (realtimeDataExtractionEndTime % TimePartitionUtils.getTimePartitionInterval() == 0) + TimePartitionUtils.isTimePartitionStartTime(realtimeDataExtractionEndTime) ? TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime) : TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime) - 1; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeDurationAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeDurationAccumulator.java index eebd8151e786..d10853db87b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeDurationAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeDurationAccumulator.java @@ -87,7 +87,7 @@ public void outputFinal(ColumnBuilder tsBlockBuilder) { if (!initResult) { tsBlockBuilder.appendNull(); } else { - tsBlockBuilder.writeLong(maxTime - minTime); + tsBlockBuilder.writeLong(saturatingTimeDifference(maxTime, minTime)); } } @@ -127,4 +127,9 @@ protected void updateMinTime(long curTime) { initResult = true; minTime = Math.min(minTime, curTime); } + + private static long saturatingTimeDifference(long maxTime, long minTime) { + long timeDifference = maxTime - minTime; + return timeDifference < 0 ? Long.MAX_VALUE : timeDifference; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/AggrWindowIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/AggrWindowIterator.java index e9847f814cff..a748fb4293ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/AggrWindowIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/AggrWindowIterator.java @@ -27,6 +27,8 @@ import java.time.ZoneId; +import static com.google.common.math.LongMath.saturatedAdd; + /** * This class iteratively generates aggregated time windows. * @@ -86,7 +88,7 @@ private TimeRange getLeftmostTimeRange() { retEndTime = Math.min(DateTimeUtils.calcPositiveIntervalByMonth(startTime, interval, zoneId), endTime); } else { - retEndTime = Math.min(startTime + interval.nonMonthDuration, endTime); + retEndTime = Math.min(saturatedAdd(startTime, interval.nonMonthDuration), endTime); } return new TimeRange(startTime, retEndTime); } @@ -94,15 +96,14 @@ private TimeRange getLeftmostTimeRange() { private TimeRange getRightmostTimeRange() { long retStartTime; long retEndTime; - long queryRange = endTime - startTime; long intervalNum; if (slidingStep.containsMonth()) { intervalNum = - (long) - Math.ceil( - (double) queryRange - / (slidingStep.getMaxTotalDuration(TimestampPrecisionUtils.currPrecision))); + ITimeRangeIterator.ceilDivTimeRange( + startTime, + endTime, + slidingStep.getMaxTotalDuration(TimestampPrecisionUtils.currPrecision)); long tempRetStartTime = DateTimeUtils.calcPositiveIntervalByMonth( startTime, slidingStep.multiple(intervalNum - 1), zoneId); @@ -116,8 +117,11 @@ private TimeRange getRightmostTimeRange() { } intervalNum -= 1; } else { - intervalNum = (long) Math.ceil(queryRange / (double) slidingStep.nonMonthDuration); - retStartTime = slidingStep.nonMonthDuration * (intervalNum - 1) + startTime; + intervalNum = + ITimeRangeIterator.ceilDivTimeRange(startTime, endTime, slidingStep.nonMonthDuration); + retStartTime = + ITimeRangeIterator.rightmostTimeRangeStart( + startTime, endTime, slidingStep.nonMonthDuration); } if (interval.containsMonth()) { @@ -129,7 +133,7 @@ private TimeRange getRightmostTimeRange() { startTime, interval.merge(slidingStep.multiple(intervalNum - 1)), zoneId), endTime); } else { - retEndTime = Math.min(retStartTime + interval.nonMonthDuration, endTime); + retEndTime = Math.min(saturatedAdd(retStartTime, interval.nonMonthDuration), endTime); } return new TimeRange(retStartTime, retEndTime); } @@ -155,6 +159,10 @@ public boolean hasNextTimeRange() { DateTimeUtils.calcPositiveIntervalByMonth( startTime, slidingStep.multiple(timeRangeCount), zoneId); } else { + if (!ITimeRangeIterator.canMoveForward( + curStartTime, slidingStep.nonMonthDuration, endTime)) { + return false; + } retStartTime = curStartTime + slidingStep.nonMonthDuration; } // This is an open interval , [0-100) @@ -167,6 +175,10 @@ public boolean hasNextTimeRange() { throw new UnsupportedOperationException( "Ascending is not supported when sliding step contains month."); } else { + if (!ITimeRangeIterator.canMoveBackward( + curStartTime, slidingStep.nonMonthDuration, startTime)) { + return false; + } retStartTime = curStartTime - slidingStep.nonMonthDuration; } if (retStartTime < startTime) { @@ -179,7 +191,7 @@ public boolean hasNextTimeRange() { DateTimeUtils.calcPositiveIntervalByMonth( startTime, slidingStep.multiple(timeRangeCount).merge(interval), zoneId); } else { - retEndTime = retStartTime + interval.nonMonthDuration; + retEndTime = saturatedAdd(retStartTime, interval.nonMonthDuration); } retEndTime = Math.min(retEndTime, endTime); curTimeRange = new TimeRange(retStartTime, retEndTime); @@ -209,15 +221,14 @@ public long currentOutputTime() { @Override public long getTotalIntervalNum() { - long queryRange = endTime - startTime; long intervalNum; if (slidingStep.containsMonth()) { intervalNum = - (long) - Math.ceil( - (double) queryRange - / (slidingStep.getMaxTotalDuration(TimestampPrecisionUtils.currPrecision))); + ITimeRangeIterator.ceilDivTimeRange( + startTime, + endTime, + slidingStep.getMaxTotalDuration(TimestampPrecisionUtils.currPrecision)); long retStartTime = DateTimeUtils.calcPositiveIntervalByMonth( startTime, slidingStep.multiple(intervalNum), zoneId); @@ -228,7 +239,8 @@ public long getTotalIntervalNum() { startTime, slidingStep.multiple(intervalNum), zoneId); } } else { - intervalNum = (long) Math.ceil(queryRange / (double) slidingStep.nonMonthDuration); + intervalNum = + ITimeRangeIterator.ceilDivTimeRange(startTime, endTime, slidingStep.nonMonthDuration); } return intervalNum; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/ITimeRangeIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/ITimeRangeIterator.java index 9dbd2ce77a30..fe1a3031a5a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/ITimeRangeIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/ITimeRangeIterator.java @@ -21,6 +21,10 @@ import org.apache.tsfile.read.common.TimeRange; +import java.math.BigInteger; + +import static com.google.common.math.LongMath.saturatedAdd; + /** * This interface used for iteratively generating aggregated time windows in GROUP BY query. * @@ -47,8 +51,43 @@ public interface ITimeRangeIterator { default TimeRange getFinalTimeRange(TimeRange timeRange, boolean leftCRightO) { return leftCRightO - ? new TimeRange(timeRange.getMin(), timeRange.getMax() - 1) - : new TimeRange(timeRange.getMin() + 1, timeRange.getMax()); + ? new TimeRange(timeRange.getMin(), saturatedAdd(timeRange.getMax(), -1)) + : new TimeRange(saturatedAdd(timeRange.getMin(), 1), timeRange.getMax()); + } + + static boolean canMoveForward(long current, long step, long upperBound) { + return step > 0 && current <= Long.MAX_VALUE - step && current + step < upperBound; + } + + static boolean canMoveBackward(long current, long step, long lowerBound) { + return step > 0 && current >= Long.MIN_VALUE + step && current - step >= lowerBound; + } + + static long ceilDivTimeRange(long startTime, long endTime, long divisor) { + BigInteger range = + BigInteger.valueOf(endTime) + .subtract(BigInteger.valueOf(startTime)) + .add(BigInteger.valueOf(divisor).subtract(BigInteger.ONE)); + return range + .divide(BigInteger.valueOf(divisor)) + .min(BigInteger.valueOf(Long.MAX_VALUE)) + .longValue(); + } + + static long rightmostTimeRangeStart(long startTime, long endTime, long slidingStep) { + BigInteger distanceMinusOne = + BigInteger.valueOf(endTime) + .subtract(BigInteger.valueOf(startTime)) + .subtract(BigInteger.ONE); + long remainder = distanceMinusOne.mod(BigInteger.valueOf(slidingStep)).longValue(); + return saturatedAdd(saturatedAdd(endTime, -1), -remainder); + } + + static boolean isTimeRangeDistanceGreaterThan(long startTime, long endTime, long distance) { + return BigInteger.valueOf(endTime) + .subtract(BigInteger.valueOf(startTime)) + .compareTo(BigInteger.valueOf(distance)) + > 0; } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowIterator.java index df07e15d032e..ac3a53fc7109 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowIterator.java @@ -21,6 +21,10 @@ import org.apache.tsfile.read.common.TimeRange; +import java.math.BigInteger; + +import static com.google.common.math.LongMath.saturatedAdd; + /** * This class iteratively generates pre-aggregated time windows. * @@ -71,7 +75,7 @@ public TimeRange getFirstTimeRange() { } private TimeRange getLeftmostTimeRange() { - long retEndTime = Math.min(startTime + curInterval, endTime); + long retEndTime = Math.min(saturatedAdd(startTime, curInterval), endTime); updateIntervalAndStep(); return new TimeRange(startTime, retEndTime); } @@ -79,13 +83,14 @@ private TimeRange getLeftmostTimeRange() { private TimeRange getRightmostTimeRange() { long retStartTime; long retEndTime; - long intervalNum = (long) Math.ceil((endTime - startTime) / (double) slidingStep); - retStartTime = slidingStep * (intervalNum - 1) + startTime; - if (isIntervalCyclicChange && endTime - retStartTime > interval % slidingStep) { - retStartTime += interval % slidingStep; + retStartTime = ITimeRangeIterator.rightmostTimeRangeStart(startTime, endTime, slidingStep); + if (isIntervalCyclicChange + && ITimeRangeIterator.isTimeRangeDistanceGreaterThan( + retStartTime, endTime, interval % slidingStep)) { + retStartTime = saturatedAdd(retStartTime, interval % slidingStep); updateIntervalAndStep(); } - retEndTime = Math.min(retStartTime + curInterval, endTime); + retEndTime = Math.min(saturatedAdd(retStartTime, curInterval), endTime); updateIntervalAndStep(); return new TimeRange(retStartTime, retEndTime); } @@ -105,18 +110,24 @@ public boolean hasNextTimeRange() { long retEndTime; long curStartTime = curTimeRange.getMin(); if (isAscending) { + if (!ITimeRangeIterator.canMoveForward(curStartTime, curSlidingStep, endTime)) { + return false; + } retStartTime = curStartTime + curSlidingStep; // This is an open interval , [0-100) if (retStartTime >= endTime) { return false; } } else { + if (!ITimeRangeIterator.canMoveBackward(curStartTime, curSlidingStep, startTime)) { + return false; + } retStartTime = curStartTime - curSlidingStep; if (retStartTime < startTime) { return false; } } - retEndTime = Math.min(retStartTime + curInterval, endTime); + retEndTime = Math.min(saturatedAdd(retStartTime, curInterval), endTime); updateIntervalAndStep(); curTimeRange = new TimeRange(retStartTime, retEndTime); hasCachedTimeRange = true; @@ -177,19 +188,24 @@ public long currentOutputTime() { @Override public long getTotalIntervalNum() { - long queryRange = endTime - startTime; if (slidingStep >= interval || interval % slidingStep == 0) { - return (long) Math.ceil(queryRange / (double) slidingStep); + return ITimeRangeIterator.ceilDivTimeRange(startTime, endTime, slidingStep); } long interval1 = interval % slidingStep; long interval2 = slidingStep - interval % slidingStep; - long intervalNum = Math.floorDiv(queryRange, interval1 + interval2); - long tmpStartTime = startTime + intervalNum * (interval1 + interval2); - if (tmpStartTime + interval1 > endTime) { - return intervalNum * 2 + 1; - } else { - return intervalNum * 2 + 2; - } + BigInteger queryRange = BigInteger.valueOf(endTime).subtract(BigInteger.valueOf(startTime)); + BigInteger intervalNum = queryRange.divide(BigInteger.valueOf(interval1 + interval2)); + BigInteger result = + intervalNum + .multiply(BigInteger.valueOf(2)) + .add( + queryRange + .remainder(BigInteger.valueOf(interval1 + interval2)) + .compareTo(BigInteger.valueOf(interval1)) + < 0 + ? BigInteger.ONE + : BigInteger.valueOf(2)); + return result.min(BigInteger.valueOf(Long.MAX_VALUE)).longValue(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowWithNaturalMonthIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowWithNaturalMonthIterator.java index a4a0aa626114..29ed2391ba3d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowWithNaturalMonthIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowWithNaturalMonthIterator.java @@ -26,6 +26,8 @@ import java.time.ZoneId; +import static com.google.common.math.LongMath.saturatedAdd; + public class PreAggrWindowWithNaturalMonthIterator implements ITimeRangeIterator { private static final int HEAP_MAX_SIZE = 100; @@ -109,12 +111,12 @@ private void initHeap() { TimeRange firstTimeRange = aggrWindowIterator.nextTimeRange(); if (leftCRightO) { timeBoundaryHeap.add(firstTimeRange.getMin()); - timeBoundaryHeap.add(firstTimeRange.getMax() + 1); + timeBoundaryHeap.add(saturatedAdd(firstTimeRange.getMax(), 1)); curStartTimeForIterator = firstTimeRange.getMin(); } else { - timeBoundaryHeap.add(firstTimeRange.getMin() - 1); + timeBoundaryHeap.add(saturatedAdd(firstTimeRange.getMin(), -1)); timeBoundaryHeap.add(firstTimeRange.getMax()); - curStartTimeForIterator = firstTimeRange.getMin() - 1; + curStartTimeForIterator = saturatedAdd(firstTimeRange.getMin(), -1); } tryToExpandHeap(); } @@ -125,12 +127,12 @@ private void tryToExpandHeap() { timeRangeToExpand = aggrWindowIterator.nextTimeRange(); if (leftCRightO) { timeBoundaryHeap.add(timeRangeToExpand.getMin()); - timeBoundaryHeap.add(timeRangeToExpand.getMax() + 1); + timeBoundaryHeap.add(saturatedAdd(timeRangeToExpand.getMax(), 1)); curStartTimeForIterator = timeRangeToExpand.getMin(); } else { - timeBoundaryHeap.add(timeRangeToExpand.getMin() - 1); + timeBoundaryHeap.add(saturatedAdd(timeRangeToExpand.getMin(), -1)); timeBoundaryHeap.add(timeRangeToExpand.getMax()); - curStartTimeForIterator = timeRangeToExpand.getMin() - 1; + curStartTimeForIterator = saturatedAdd(timeRangeToExpand.getMin(), -1); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperator.java index 9bdb57dc5bd0..ecdcfa643b66 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperator.java @@ -45,6 +45,7 @@ import org.apache.tsfile.read.common.block.column.TsBlockSerde; import org.apache.tsfile.utils.RamUsageEstimator; +import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.HashMap; @@ -58,6 +59,9 @@ public class InferenceOperator implements ProcessOperator { + private static final String GENERATED_TIME_COLUMN_OUT_OF_RANGE_MESSAGE = + "Generated time column is out of range."; + private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(InferenceOperator.class); @@ -160,7 +164,7 @@ private void fillTimeColumn(TsBlock tsBlock) { Column timeColumn = tsBlock.getTimeColumn(); long[] time = timeColumn.getLongs(); for (int i = 0; i < time.length; i++) { - time[i] = maxTimestamp + interval * currentRowIndex; + time[i] = calculateGeneratedTime(maxTimestamp, interval, currentRowIndex); currentRowIndex++; } } @@ -296,7 +300,7 @@ private TsBlock preProcess(TsBlock inputTsBlock) { private void submitInferenceTask() { if (generateTimeColumn) { - interval = (maxTimestamp - minTimestamp) / totalRow; + interval = calculateGeneratedTimeInterval(minTimestamp, maxTimestamp, totalRow); } TsBlock inputTsBlock = inputTsBlockBuilder.build(); @@ -374,4 +378,34 @@ public long ramBytesUsed() { ? 0 : targetColumnNames.stream().mapToLong(RamUsageEstimator::sizeOf).sum()); } + + static long calculateGeneratedTimeInterval(long minTimestamp, long maxTimestamp, long totalRow) { + try { + BigInteger interval = + BigInteger.valueOf(maxTimestamp) + .subtract(BigInteger.valueOf(minTimestamp)) + .divide(BigInteger.valueOf(totalRow)); + if (interval.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + throw new ArithmeticException(); + } + return interval.longValue(); + } catch (ArithmeticException e) { + throw new ModelInferenceProcessException(GENERATED_TIME_COLUMN_OUT_OF_RANGE_MESSAGE); + } + } + + static long calculateGeneratedTime(long maxTimestamp, long interval, long currentRowIndex) { + try { + BigInteger generatedTime = + BigInteger.valueOf(maxTimestamp) + .add(BigInteger.valueOf(interval).multiply(BigInteger.valueOf(currentRowIndex))); + if (generatedTime.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0 + || generatedTime.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + throw new ArithmeticException(); + } + return generatedTime.longValue(); + } catch (ArithmeticException e) { + throw new ModelInferenceProcessException(GENERATED_TIME_COLUMN_OUT_OF_RANGE_MESSAGE); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java index 840148438a7d..8747b4e21575 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java @@ -21,6 +21,8 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFillFilter; +import static org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceLessThanOrEqualTo; + public class FixedIntervalFillFilter implements IFillFilter { // the time precision of this field is same as the system time_precision configuration. @@ -34,6 +36,6 @@ public FixedIntervalFillFilter(long timeInterval) { public boolean needFill(long time, long previousTime) { // the reason that we use Math.abs is that we may use order by time desc which will cause // previousTime is larger than time - return Math.abs(time - previousTime) <= timeInterval; + return isTimeDistanceLessThanOrEqualTo(time, previousTime, timeInterval); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/linear/LinearFill.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/linear/LinearFill.java index d35ec87d01b3..ee209b906be3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/linear/LinearFill.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/linear/LinearFill.java @@ -26,6 +26,7 @@ import org.apache.tsfile.read.common.block.column.TimeColumn; import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.iotdb.commons.utils.CommonDateTimeUtils.timeDifferenceAsDouble; /** * The result of Linear Fill functions at timestamp "T" is calculated by performing a linear fitting @@ -135,9 +136,8 @@ private boolean fill( } private double getFactor(long currentTime) { - return nextTimeInCurrentColumn - previousTime == 0 - ? 0.0 - : ((double) (currentTime - previousTime)) / (nextTimeInCurrentColumn - previousTime); + double timeRange = timeDifferenceAsDouble(nextTimeInCurrentColumn, previousTime); + return timeRange == 0 ? 0.0 : timeDifferenceAsDouble(currentTime, previousTime) / timeRange; } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 1536c3cc8656..c3d48812a462 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -2226,7 +2226,7 @@ static void analyzeGroupByTime(Analysis analysis, QueryStatement queryStatement) analysis.setGroupByTimeParameter(groupByTimeParameter); Expression globalTimePredicate = analysis.getGlobalTimePredicate(); - Expression groupByTimePredicate = ExpressionFactory.groupByTime(groupByTimeParameter); + Expression groupByTimePredicate = getGroupByTimePredicate(groupByTimeParameter); if (globalTimePredicate == null) { globalTimePredicate = groupByTimePredicate; } else { @@ -2235,6 +2235,24 @@ static void analyzeGroupByTime(Analysis analysis, QueryStatement queryStatement) analysis.setGlobalTimePredicate(globalTimePredicate); } + private static Expression getGroupByTimePredicate(GroupByTimeParameter groupByTimeParameter) { + if (groupByTimeParameter.isLeftCRightO() + || groupByTimeParameter.getEndTime() != Long.MAX_VALUE) { + return ExpressionFactory.groupByTime(groupByTimeParameter); + } + GroupByTimeParameter rightOpenParameter = + new GroupByTimeParameter( + groupByTimeParameter.getStartTime() + 1, + groupByTimeParameter.getEndTime(), + groupByTimeParameter.getInterval(), + groupByTimeParameter.getSlidingStep(), + true); + return ExpressionFactory.or( + ExpressionFactory.groupByTime(rightOpenParameter), + ExpressionFactory.eq( + ExpressionFactory.time(), ExpressionFactory.longValue(Long.MAX_VALUE))); + } + static void analyzeFill(Analysis analysis, QueryStatement queryStatement) { if (queryStatement.getFillComponent() == null) { return; @@ -2312,7 +2330,7 @@ public static Pair, Pair> getTimePart // (-oo, +oo) return new Pair<>(Collections.emptyList(), new Pair<>(true, true)); } - List timeRangeList = timeFilter.getTimeRanges(); + List timeRangeList = normalizeTimeRanges(timeFilter.getTimeRanges()); if (timeRangeList.isEmpty()) { // no satisfied time range return new Pair<>(Collections.emptyList(), new Pair<>(false, false)); @@ -2363,11 +2381,7 @@ public static Pair, Pair> getTimePart result.add(timePartitionSlot); // next init timePartitionSlot = new TTimePartitionSlot(endTime); - // beware of overflow - endTime = - endTime + TimePartitionUtils.getTimePartitionInterval() > endTime - ? endTime + TimePartitionUtils.getTimePartitionInterval() - : Long.MAX_VALUE; + endTime = TimePartitionUtils.getTimePartitionUpperBound(endTime); } else { index++; if (index < size) { @@ -2396,8 +2410,39 @@ private static void reserveMemoryForTimePartitionSlot( return; } long size = TimePartitionUtils.getEstimateTimePartitionSize(minTime, maxTime); - context.reserveMemoryForFrontEnd( - RamUsageEstimator.shallowSizeOfInstance(TTimePartitionSlot.class) * size); + context.reserveMemoryForFrontEnd(estimateTimePartitionSlotMemory(size)); + } + + private static List normalizeTimeRanges(List timeRanges) { + if (timeRanges.size() < 2) { + return timeRanges; + } + + List normalized = new ArrayList<>(); + TimeRange current = timeRanges.get(0); + for (int i = 1; i < timeRanges.size(); i++) { + TimeRange next = timeRanges.get(i); + // Time ranges returned by a Filter are ordered. Merge both overlapping and adjacent ranges + // so partition routing does not depend on the shape of an OR expression. + boolean overlaps = next.getMin() <= current.getMax(); + boolean adjacent = + current.getMax() != Long.MAX_VALUE && next.getMin() == current.getMax() + 1; + if (overlaps || adjacent) { + current = new TimeRange(current.getMin(), Math.max(current.getMax(), next.getMax())); + } else { + normalized.add(current); + current = next; + } + } + normalized.add(current); + return normalized; + } + + static long estimateTimePartitionSlotMemory(long timePartitionSlotCount) { + long timePartitionSlotSize = RamUsageEstimator.shallowSizeOfInstance(TTimePartitionSlot.class); + return timePartitionSlotCount > Long.MAX_VALUE / timePartitionSlotSize + ? Long.MAX_VALUE + : timePartitionSlotSize * timePartitionSlotCount; } private void analyzeInto( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/ExpressionFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/ExpressionFactory.java index 58e21ed814b5..c081e0f07c26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/ExpressionFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/ExpressionFactory.java @@ -216,13 +216,27 @@ public static BetweenExpression notBetween( } public static GroupByTimeExpression groupByTime(GroupByTimeParameter parameter) { + if (!parameter.isLeftCRightO() && parameter.getEndTime() == Long.MAX_VALUE) { + throw new IllegalArgumentException( + "Right-closed GROUP BY TIME with Long.MAX_VALUE end time cannot be represented " + + "as a single right-open time filter."); + } long startTime = - parameter.isLeftCRightO() ? parameter.getStartTime() : parameter.getStartTime() + 1; - long endTime = parameter.isLeftCRightO() ? parameter.getEndTime() : parameter.getEndTime() + 1; + parameter.isLeftCRightO() + ? parameter.getStartTime() + : saturatingIncrement(parameter.getStartTime()); + long endTime = + parameter.isLeftCRightO() + ? parameter.getEndTime() + : saturatingIncrement(parameter.getEndTime()); return new GroupByTimeExpression( startTime, endTime, parameter.getInterval(), parameter.getSlidingStep()); } + private static long saturatingIncrement(long value) { + return value == Long.MAX_VALUE ? Long.MAX_VALUE : value + 1; + } + public static GroupByTimeExpression groupByTime( long startTime, long endTime, long interval, long slidingStep) { return new GroupByTimeExpression( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java index 16624e203f43..a4bd835f8ba2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java @@ -46,6 +46,7 @@ import org.apache.tsfile.utils.TimeDuration; +import java.math.BigInteger; import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; @@ -53,6 +54,8 @@ import java.util.TimeZone; import java.util.concurrent.TimeUnit; +import static org.apache.iotdb.commons.utils.CommonDateTimeUtils.saturateToLong; + /** * Optimization phase: Distributed plan planning. * @@ -67,6 +70,9 @@ */ public class LimitOffsetPushDown implements PlanOptimizer { + private static final BigInteger BIG_INTEGER_MAX = BigInteger.valueOf(Integer.MAX_VALUE); + private static final BigInteger BIG_INTEGER_MIN = BigInteger.valueOf(Integer.MIN_VALUE); + @Override public PlanNode optimize(PlanNode plan, Analysis analysis, MPPQueryContext context) { if (analysis.getStatement().getType() != StatementType.QUERY) { @@ -283,8 +289,8 @@ private static void pushDownLimitOffsetToTimeParameterContainingMonth( // Evaluate the day of month as 28 days long totalStep = slidingStep.getMinTotalDuration(TimeUnit.MILLISECONDS); - long size = (endTime - startTime + totalStep - 1) / totalStep; - if (size > offsetSize) { + BigInteger size = ceilDivTimeRange(startTime, endTime, totalStep); + if (size.compareTo(BigInteger.valueOf(offsetSize)) > 0) { TimeZone timeZone = TimeZone.getTimeZone(zoneId); // ordering in group by month must be ascending long newStartTime = @@ -314,10 +320,19 @@ private static void pushDownLimitOffsetToTimeParameterContainingMonth( private static TimeDuration calculateEndTimeDuration( TimeDuration slidingStep, TimeDuration interval, long limitSize, long offsetSize) { - long length = offsetSize + limitSize - 1; + BigInteger length = + BigInteger.valueOf(offsetSize).add(BigInteger.valueOf(limitSize)).subtract(BigInteger.ONE); // startTime + offsetSize * step + (limitSize - 1) * step + interval - int monthDuration = (int) (length * slidingStep.monthDuration + interval.monthDuration); - long nonMonthDuration = length * slidingStep.nonMonthDuration + interval.nonMonthDuration; + int monthDuration = + saturateToInt( + length + .multiply(BigInteger.valueOf(slidingStep.monthDuration)) + .add(BigInteger.valueOf(interval.monthDuration))); + long nonMonthDuration = + saturateToLong( + length + .multiply(BigInteger.valueOf(slidingStep.nonMonthDuration)) + .add(BigInteger.valueOf(interval.nonMonthDuration))); return new TimeDuration(monthDuration, nonMonthDuration); } @@ -336,18 +351,26 @@ public static void pushDownLimitOffsetToTimeParameter( long interval = groupByTimeComponent.getInterval().nonMonthDuration; long limitSize = queryStatement.getRowLimit(); long offsetSize = queryStatement.getRowOffset(); - long size = (endTime - startTime + step - 1) / step; - if (size > offsetSize) { + BigInteger size = ceilDivTimeRange(startTime, endTime, step); + if (size.compareTo(BigInteger.valueOf(offsetSize)) > 0) { if (queryStatement.getResultTimeOrder() == Ordering.ASC) { - startTime = startTime + offsetSize * step; + startTime = addTimeDuration(startTime, BigInteger.valueOf(offsetSize), step, 0); } else { - long startTimeInterval = size - offsetSize - limitSize; - startTime = startTime + (startTimeInterval < 0 ? 0 : startTimeInterval) * step; + BigInteger startTimeInterval = + size.subtract(BigInteger.valueOf(offsetSize)).subtract(BigInteger.valueOf(limitSize)); + startTime = + addTimeDuration( + startTime, + startTimeInterval.signum() < 0 ? BigInteger.ZERO : startTimeInterval, + step, + 0); } endTime = limitSize == 0 ? endTime - : Math.min(endTime, startTime + (limitSize - 1) * step + interval); + : Math.min( + endTime, + addTimeDuration(startTime, BigInteger.valueOf(limitSize - 1), step, interval)); groupByTimeComponent.setEndTime(endTime); groupByTimeComponent.setStartTime(startTime); } else { @@ -386,8 +409,11 @@ public static List pushDownLimitOffsetInGroupByTimeForDevice( long startTime = groupByTimeComponent.getStartTime(); long endTime = groupByTimeComponent.getEndTime(); long slidingStep = groupByTimeComponent.getSlidingStep().nonMonthDuration; - long size = (endTime - startTime + slidingStep - 1) / slidingStep; - if (size == 0 || size * deviceNames.size() <= queryStatement.getRowOffset()) { + BigInteger size = ceilDivTimeRange(startTime, endTime, slidingStep); + if (size.signum() == 0 + || size.multiply(BigInteger.valueOf(deviceNames.size())) + .compareTo(BigInteger.valueOf(queryStatement.getRowOffset())) + <= 0) { // resultSet is empty queryStatement.setResultSetEmpty(true); return deviceNames; @@ -396,19 +422,20 @@ public static List pushDownLimitOffsetInGroupByTimeForDevice( long limitSize = queryStatement.getRowLimit(); long offsetSize = queryStatement.getRowOffset(); List optimizedDeviceNames = new ArrayList<>(); - int startDeviceIndex = (int) (offsetSize / size); + int startDeviceIndex = saturateToInt(BigInteger.valueOf(offsetSize).divide(size)); int endDeviceIndex = limitSize == 0 ? deviceNames.size() - 1 - : (int) - ((limitSize - ((startDeviceIndex + 1) * size - offsetSize) + size - 1) / size - + startDeviceIndex); + : calculateEndDeviceIndex(size, limitSize, offsetSize, startDeviceIndex); int index = 0; while (index < startDeviceIndex) { index++; } - queryStatement.setRowOffset(offsetSize - startDeviceIndex * size); + queryStatement.setRowOffset( + saturateToLong( + BigInteger.valueOf(offsetSize) + .subtract(BigInteger.valueOf(startDeviceIndex).multiply(size)))); // if only refer to one device, optimize the time parameter if (startDeviceIndex == endDeviceIndex) { @@ -428,4 +455,44 @@ public static List pushDownLimitOffsetInGroupByTimeForDevice( private static boolean hasLimitOffset(QueryStatement queryStatement) { return queryStatement.hasLimit() || queryStatement.hasOffset(); } + + private static BigInteger ceilDivTimeRange(long startTime, long endTime, long divisor) { + return BigInteger.valueOf(endTime) + .subtract(BigInteger.valueOf(startTime)) + .add(BigInteger.valueOf(divisor).subtract(BigInteger.ONE)) + .divide(BigInteger.valueOf(divisor)); + } + + private static long addTimeDuration( + long startTime, BigInteger stepCount, long step, long interval) { + return saturateToLong( + BigInteger.valueOf(startTime) + .add(stepCount.multiply(BigInteger.valueOf(step))) + .add(BigInteger.valueOf(interval))); + } + + private static int calculateEndDeviceIndex( + BigInteger size, long limitSize, long offsetSize, int startDeviceIndex) { + BigInteger firstDeviceRemaining = + BigInteger.valueOf(startDeviceIndex + 1L) + .multiply(size) + .subtract(BigInteger.valueOf(offsetSize)); + return saturateToInt( + BigInteger.valueOf(limitSize) + .subtract(firstDeviceRemaining) + .add(size) + .subtract(BigInteger.ONE) + .divide(size) + .add(BigInteger.valueOf(startDeviceIndex))); + } + + private static int saturateToInt(BigInteger value) { + if (value.compareTo(BIG_INTEGER_MAX) > 0) { + return Integer.MAX_VALUE; + } + if (value.compareTo(BIG_INTEGER_MIN) < 0) { + return Integer.MIN_VALUE; + } + return value.intValue(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index d000f62de327..31f5b828fd3e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -1641,7 +1641,8 @@ private GroupByTimeComponent parseGroupByTimeClause( TimeDuration slidingStep = groupByTimeComponent.getSlidingStep(); if (slidingStep.containsMonth() && Math.ceil( - ((groupByTimeComponent.getEndTime() - groupByTimeComponent.getStartTime()) + (((double) groupByTimeComponent.getEndTime() + - (double) groupByTimeComponent.getStartTime()) / (double) slidingStep.getMinTotalDuration(currPrecision))) >= 10000) { throw new SemanticException("The time windows may exceed 10000, please ensure your input."); @@ -2753,20 +2754,25 @@ private TimeRange parseDeleteTimeRange(Expression predicate) { parseDeleteTimeRange(((LogicAndExpression) predicate).getLeftExpression()); TimeRange rightTimeRange = parseDeleteTimeRange(((LogicAndExpression) predicate).getRightExpression()); - return new TimeRange( - Math.max(leftTimeRange.getMin(), rightTimeRange.getMin()), - Math.min(leftTimeRange.getMax(), rightTimeRange.getMax())); + long min = Math.max(leftTimeRange.getMin(), rightTimeRange.getMin()); + long max = Math.min(leftTimeRange.getMax(), rightTimeRange.getMax()); + if (min > max) { + throw new SemanticException(DELETE_RANGE_ERROR_MSG); + } + return new TimeRange(min, max); } else if (predicate instanceof CompareBinaryExpression) { if (((CompareBinaryExpression) predicate).getLeftExpression() instanceof TimestampOperand) { return parseTimeRangeForDeleteTimeRange( predicate.getExpressionType(), ((CompareBinaryExpression) predicate).getLeftExpression(), - ((CompareBinaryExpression) predicate).getRightExpression()); + ((CompareBinaryExpression) predicate).getRightExpression(), + predicate); } else { return parseTimeRangeForDeleteTimeRange( predicate.getExpressionType(), ((CompareBinaryExpression) predicate).getRightExpression(), - ((CompareBinaryExpression) predicate).getLeftExpression()); + ((CompareBinaryExpression) predicate).getLeftExpression(), + predicate); } } else { throw new SemanticException(DELETE_RANGE_ERROR_MSG); @@ -2774,7 +2780,10 @@ private TimeRange parseDeleteTimeRange(Expression predicate) { } private TimeRange parseTimeRangeForDeleteTimeRange( - ExpressionType expressionType, Expression timeExpression, Expression valueExpression) { + ExpressionType expressionType, + Expression timeExpression, + Expression valueExpression, + Expression comparisonExpression) { if (!(timeExpression instanceof TimestampOperand) || !(valueExpression instanceof ConstantOperand)) { throw new SemanticException(DELETE_ONLY_SUPPORT_TIME_EXP_ERROR_MSG); @@ -2787,10 +2796,20 @@ private TimeRange parseTimeRangeForDeleteTimeRange( long time = Long.parseLong(((ConstantOperand) valueExpression).getValueString()); switch (expressionType) { case LESS_THAN: + if (time == Long.MIN_VALUE) { + throw new SemanticException( + String.format( + "The time predicate does not select any time range: %s", comparisonExpression)); + } return new TimeRange(Long.MIN_VALUE, time - 1); case LESS_EQUAL: return new TimeRange(Long.MIN_VALUE, time); case GREATER_THAN: + if (time == Long.MAX_VALUE) { + throw new SemanticException( + String.format( + "The time predicate does not select any time range: %s", comparisonExpression)); + } return new TimeRange(time + 1, Long.MAX_VALUE); case GREATER_EQUAL: return new TimeRange(time, Long.MAX_VALUE); @@ -3233,14 +3252,17 @@ public Long parseDateExpression(IoTDBSqlParser.DateExpressionContext ctx, String long time; time = parseDateTimeFormat(ctx.getChild(0).getText()); for (int i = 1; i < ctx.getChildCount(); i = i + 2) { - if ("+".equals(ctx.getChild(i).getText())) { - time += - DateTimeUtils.convertDurationStrToLong( - time, ctx.getChild(i + 1).getText(), precision, false); - } else { - time -= + try { + long duration = DateTimeUtils.convertDurationStrToLong( time, ctx.getChild(i + 1).getText(), precision, false); + time = + "+".equals(ctx.getChild(i).getText()) + ? Math.addExact(time, duration) + : Math.subtractExact(time, duration); + } catch (ArithmeticException e) { + throw new SemanticException( + String.format("Date expression is out of range: %s", ctx.getText())); } } return time; @@ -3250,10 +3272,16 @@ private Long parseDateExpression(IoTDBSqlParser.DateExpressionContext ctx, long long time; time = parseDateTimeFormat(ctx.getChild(0).getText(), currentTime); for (int i = 1; i < ctx.getChildCount(); i = i + 2) { - if ("+".equals(ctx.getChild(i).getText())) { - time += DateTimeUtils.convertDurationStrToLong(time, ctx.getChild(i + 1).getText(), false); - } else { - time -= DateTimeUtils.convertDurationStrToLong(time, ctx.getChild(i + 1).getText(), false); + try { + long duration = + DateTimeUtils.convertDurationStrToLong(time, ctx.getChild(i + 1).getText(), false); + time = + "+".equals(ctx.getChild(i).getText()) + ? Math.addExact(time, duration) + : Math.subtractExact(time, duration); + } catch (ArithmeticException e) { + throw new SemanticException( + String.format("Date expression is out of range: %s", ctx.getText())); } } return time; @@ -4131,6 +4159,11 @@ public GetRegionIdStatement parseTimeRangeExpression( Math.max(getRegionIdStatement.getStartTimeStamp(), timestamp)); break; case GREATER_THAN: + if (timestamp == Long.MAX_VALUE) { + throw new SemanticException( + String.format( + "The time predicate does not select any time range: %s", timeRangeExpression)); + } getRegionIdStatement.setStartTimeStamp( Math.max(getRegionIdStatement.getStartTimeStamp(), timestamp + 1)); break; @@ -4139,6 +4172,11 @@ public GetRegionIdStatement parseTimeRangeExpression( Math.min(getRegionIdStatement.getEndTimeStamp(), timestamp)); break; case LESS_THAN: + if (timestamp == Long.MIN_VALUE) { + throw new SemanticException( + String.format( + "The time predicate does not select any time range: %s", timeRangeExpression)); + } getRegionIdStatement.setEndTimeStamp( Math.min(getRegionIdStatement.getEndTimeStamp(), timestamp - 1)); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index 3495b218e612..157ff5cbc3a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -222,7 +222,8 @@ public List splitByPartition(IAnalysis analysis) { // for each List in split, they are range1.start, range1.end, range2.start, range2.end, ... List ranges = new ArrayList<>(); for (int i = 1; i < rowCount; i++) { // times are sorted in session API. - if (times[i] >= upperBoundOfTimePartition) { + if (TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + times[i], timePartitionSlot.getStartTime(), upperBoundOfTimePartition)) { // a new range. ranges.add(startLoc); // included ranges.add(i); // excluded @@ -319,7 +320,8 @@ public List getTimePartitionSlots() { long upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[0]); TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[0]); for (int i = 1; i < times.length; i++) { // times are sorted in session API. - if (times[i] >= upperBoundOfTimePartition) { + if (TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + times[i], timePartitionSlot.getStartTime(), upperBoundOfTimePartition)) { result.add(timePartitionSlot); // next init upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[i]); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java index 62b5d53c56ab..219706527c2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.FilterFactory; @@ -91,12 +91,11 @@ public void setTTL(long dataTTL) { */ public static Filter updateFilterUsingTTL(Filter filter, long dataTTL) { if (dataTTL != Long.MAX_VALUE) { + long ttlLowerBound = CommonUtils.getTTLLowerBound(dataTTL); if (filter != null) { - filter = - FilterFactory.and( - filter, TimeFilterApi.gtEq(CommonDateTimeUtils.currentTime() - dataTTL)); + filter = FilterFactory.and(filter, TimeFilterApi.gtEq(ttlLowerBound)); } else { - filter = TimeFilterApi.gtEq(CommonDateTimeUtils.currentTime() - dataTTL); + filter = TimeFilterApi.gtEq(ttlLowerBound); } } return filter; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index 354856e2a240..e234e41e9347 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -196,7 +196,8 @@ public List getTimePartitionSlots() { long upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[0]); TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[0]); for (int i = 1; i < times.length; i++) { // times are sorted in session API. - if (times[i] >= upperBoundOfTimePartition) { + if (TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + times[i], timePartitionSlot.getStartTime(), upperBoundOfTimePartition)) { result.add(timePartitionSlot); // next init upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[i]); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java index 930d7d622a4b..add01210235b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java @@ -51,6 +51,8 @@ import java.util.Arrays; import java.util.List; +import static com.google.common.math.LongMath.saturatedAdd; + public class MultiInputLayer extends IntermediateLayer implements IUDFInputDataSet { private static final Logger LOGGER = LoggerFactory.getLogger(MultiInputLayer.class); @@ -396,7 +398,8 @@ public YieldableState yield() throws Exception { return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; } - long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); + long nextWindowTimeEnd = + Math.min(saturatedAdd(nextWindowTimeBegin, timeInterval), displayWindowEnd); while (currentEndTime < nextWindowTimeEnd) { final YieldableState state = udfInputDataSet.yield(); if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { @@ -465,7 +468,7 @@ public YieldableState yield() throws Exception { nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, - nextWindowTimeBegin + timeInterval - 1); + saturatedAdd(nextWindowTimeBegin, timeInterval - 1)); hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == rowRecordList.size()); return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; @@ -474,7 +477,7 @@ public YieldableState yield() throws Exception { @Override public void readyForNext() { hasCached = false; - nextWindowTimeBegin += slidingStep; + nextWindowTimeBegin = saturatedAdd(nextWindowTimeBegin, slidingStep); rowRecordList.setEvictionUpperBound(nextIndexBegin + 1); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java index a419c71c535b..6e27467f35d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java @@ -45,6 +45,8 @@ import java.io.IOException; +import static com.google.common.math.LongMath.saturatedAdd; + public class SingleInputMultiReferenceLayer extends IntermediateLayer { private static final Logger LOGGER = @@ -286,7 +288,8 @@ public YieldableState yield() throws Exception { return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; } - long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); + long nextWindowTimeEnd = + Math.min(saturatedAdd(nextWindowTimeBegin, timeInterval), displayWindowEnd); while (currentEndTime < nextWindowTimeEnd) { final YieldableState state = parentLayerReader.yield(); if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { @@ -356,7 +359,7 @@ public YieldableState yield() throws Exception { nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, - nextWindowTimeBegin + timeInterval - 1); + saturatedAdd(nextWindowTimeBegin, timeInterval - 1)); hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == tvList.size()); return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; @@ -365,7 +368,7 @@ public YieldableState yield() throws Exception { @Override public void readyForNext() { hasCached = false; - nextWindowTimeBegin += slidingStep; + nextWindowTimeBegin = saturatedAdd(nextWindowTimeBegin, slidingStep); safetyPile.moveForwardTo(nextIndexBegin + 1); tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java index fdb0779867dd..328afedf4626 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java @@ -43,6 +43,8 @@ import java.io.IOException; +import static com.google.common.math.LongMath.saturatedAdd; + public class SingleInputSingleReferenceLayer extends IntermediateLayer { private static final Logger LOGGER = @@ -209,7 +211,8 @@ public YieldableState yield() throws Exception { return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; } - long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); + long nextWindowTimeEnd = + Math.min(saturatedAdd(nextWindowTimeBegin, timeInterval), displayWindowEnd); while (currentEndTime < nextWindowTimeEnd) { final YieldableState state = parentLayerReader.yield(); if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { @@ -279,7 +282,7 @@ public YieldableState yield() throws Exception { nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, - nextWindowTimeBegin + timeInterval - 1); + saturatedAdd(nextWindowTimeBegin, timeInterval - 1)); hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == tvList.size()); return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; @@ -288,7 +291,7 @@ public YieldableState yield() throws Exception { @Override public void readyForNext() { hasCached = false; - nextWindowTimeBegin += slidingStep; + nextWindowTimeBegin = saturatedAdd(nextWindowTimeBegin, slidingStep); tvList.setEvictionUpperBound(nextIndexBegin + 1); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 67d4abf59076..c104afdb335a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -163,6 +163,8 @@ public static StorageEngine getInstance() { } private static void initTimePartition() { + TimePartitionUtils.setTimePartitionOrigin( + CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin()); TimePartitionUtils.setTimePartitionInterval( CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index ecb3e114bf28..e99600ce160f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -31,7 +31,6 @@ import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; @@ -125,6 +124,7 @@ import org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeSpaceQuotaManager; import org.apache.iotdb.db.tools.settle.TsFileAndModSettleTool; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.DateTimeUtils; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -1054,8 +1054,7 @@ public void insert(InsertRowNode insertRowNode) throws WriteProcessException { long deviceTTL = DataNodeTTLCache.getInstance().getTTL(insertRowNode.getDevicePath().getNodes()); if (!isAlive(insertRowNode.getTime(), deviceTTL)) { - throw new OutOfTTLException( - insertRowNode.getTime(), (CommonDateTimeUtils.currentTime() - deviceTTL)); + throw new OutOfTTLException(insertRowNode.getTime(), CommonUtils.getTTLLowerBound(deviceTTL)); } StorageEngine.blockInsertionIfReject(); long startTime = System.nanoTime(); @@ -1133,8 +1132,7 @@ public void insertTablet(InsertTabletNode insertTabletNode) String.format( "Insertion time [%s] is less than ttl time bound [%s]", DateTimeUtils.convertLongToDate(currTime), - DateTimeUtils.convertLongToDate( - CommonDateTimeUtils.currentTime() - deviceTTL))); + DateTimeUtils.convertLongToDate(CommonUtils.getTTLLowerBound(deviceTTL)))); loc++; noFailure = false; } else { @@ -1145,7 +1143,7 @@ public void insertTablet(InsertTabletNode insertTabletNode) if (loc == insertTabletNode.getRowCount()) { throw new OutOfTTLException( insertTabletNode.getTimes()[insertTabletNode.getTimes().length - 1], - (CommonDateTimeUtils.currentTime() - deviceTTL)); + CommonUtils.getTTLLowerBound(deviceTTL)); } // before is first start point int before = loc; @@ -1210,7 +1208,7 @@ public void insertTablet(InsertTabletNode insertTabletNode) * @return whether the given time falls in ttl */ private boolean isAlive(long time, long dataTTL) { - return dataTTL == Long.MAX_VALUE || (CommonDateTimeUtils.currentTime() - time) <= dataTTL; + return CommonUtils.isAlive(time, dataTTL); } private void initFlushTimeMap(long timePartitionId) { @@ -3776,7 +3774,7 @@ public void insert(InsertRowsOfOneDeviceNode insertRowsOfOneDeviceNode) "Insertion time [%s] is less than ttl time bound [%s]", DateTimeUtils.convertLongToDate(insertRowNode.getTime()), DateTimeUtils.convertLongToDate( - CommonDateTimeUtils.currentTime() - deviceTTL)))); + CommonUtils.getTTLLowerBound(deviceTTL))))); continue; } // init map @@ -3876,7 +3874,7 @@ public void insert(InsertRowsNode insertRowsNode) "Insertion time [%s] is less than ttl time bound [%s]", DateTimeUtils.convertLongToDate(insertRowNode.getTime()), DateTimeUtils.convertLongToDate( - CommonDateTimeUtils.currentTime() - deviceTTL)))); + CommonUtils.getTTLLowerBound(deviceTTL))))); continue; } // init map diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index de8e9575a7cc..77463737a612 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PatternTreeMap; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileReader; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; @@ -31,6 +30,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.Modification; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; @@ -222,7 +222,7 @@ public Pair nextDevice() throws IllegalPathException { timeLowerBoundForCurrentDevice = ttlForCurrentDevice == Long.MAX_VALUE ? Long.MIN_VALUE - : CommonDateTimeUtils.currentTime() - ttlForCurrentDevice; + : CommonUtils.getTTLLowerBound(ttlForCurrentDevice); return currentDevice; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java index 8439687a627c..4be944ad6cc7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java @@ -44,6 +44,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.tsfile.exception.StopReadTsFileByInterruptException; import org.apache.tsfile.file.metadata.IDeviceID; @@ -359,7 +360,7 @@ public List selectCrossSpaceTask( boolean isInsertionTask) { // TODO: (xingtanzjr) need to confirm what this ttl is used for long startTime = System.currentTimeMillis(); - long ttlLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long ttlLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); // we record the variable `candidate` here is used for selecting more than one // CrossCompactionTaskResources in this method. // Add read lock for candidate source files to avoid being deleted during the selection. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index f54af9cfcbb4..953d952c1983 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -299,10 +299,10 @@ public void initChunkMetaFromTVListsWithFakeStatistics() { (int) Math.min( MAX_NUMBER_OF_FAKE_PAGE, Math.max(1, rowNum / MAX_NUMBER_OF_POINTS_IN_FAKE_PAGE)); - long timeInterval = (chunkEndTime - chunkStartTime + 1) / pageNum; - for (int i = 0; i < pageNum; i++) { - long pageStartTime = chunkStartTime + i * timeInterval; - long pageEndTime = (i == pageNum - 1) ? chunkEndTime : (pageStartTime + timeInterval - 1); + for (long[] pageTimeRange : + MemChunkTimeRangeUtils.splitFakePageTimeRanges(chunkStartTime, chunkEndTime, pageNum)) { + long pageStartTime = pageTimeRange[0]; + long pageEndTime = pageTimeRange[1]; Statistics[] pageValueStatistics = new Statistics[dataTypes.size()]; for (int column = 0; column < dataTypes.size(); column++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkTimeRangeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkTimeRangeUtils.java new file mode 100644 index 000000000000..8eb082cdf901 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkTimeRangeUtils.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.memtable; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +final class MemChunkTimeRangeUtils { + + private MemChunkTimeRangeUtils() { + // Utility class. + } + + static List splitFakePageTimeRanges(long chunkStartTime, long chunkEndTime, int pageNum) { + BigInteger timeRange = + BigInteger.valueOf(chunkEndTime) + .subtract(BigInteger.valueOf(chunkStartTime)) + .add(BigInteger.ONE); + int effectivePageNum = + timeRange.compareTo(BigInteger.valueOf(pageNum)) < 0 ? timeRange.intValue() : pageNum; + BigInteger pageTimeInterval = timeRange.divide(BigInteger.valueOf(effectivePageNum)); + BigInteger chunkStartTimeAsBigInteger = BigInteger.valueOf(chunkStartTime); + + List pageTimeRanges = new ArrayList<>(effectivePageNum); + for (int i = 0; i < effectivePageNum; i++) { + BigInteger pageStartTime = + chunkStartTimeAsBigInteger.add(pageTimeInterval.multiply(BigInteger.valueOf(i))); + BigInteger pageEndTime = + i == effectivePageNum - 1 + ? BigInteger.valueOf(chunkEndTime) + : chunkStartTimeAsBigInteger + .add(pageTimeInterval.multiply(BigInteger.valueOf(((long) i) + 1))) + .subtract(BigInteger.ONE); + pageTimeRanges.add(new long[] {pageStartTime.longValue(), pageEndTime.longValue()}); + } + return pageTimeRanges; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java index 223e9ebb8114..9f12dac66913 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java @@ -247,10 +247,10 @@ public void initChunkMetaFromTVListsWithFakeStatistics() { (int) Math.min( MAX_NUMBER_OF_FAKE_PAGE, Math.max(1, rowNum / MAX_NUMBER_OF_POINTS_IN_FAKE_PAGE)); - long timeInterval = (chunkEndTime - chunkStartTime + 1) / pageNum; - for (int i = 0; i < pageNum; i++) { - long pageStartTime = chunkStartTime + i * timeInterval; - long pageEndTime = (i == pageNum - 1) ? chunkEndTime : (pageStartTime + timeInterval - 1); + for (long[] pageTimeRange : + MemChunkTimeRangeUtils.splitFakePageTimeRanges(chunkStartTime, chunkEndTime, pageNum)) { + long pageStartTime = pageTimeRange[0]; + long pageEndTime = pageTimeRange[1]; pageStatisticsList.add(generateFakeStatistics(dataType, pageStartTime, pageEndTime)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 7a72752af42e..6930df6b9c3b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -27,7 +27,6 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -74,6 +73,7 @@ import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager; import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.MemUtils; import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; @@ -2304,9 +2304,7 @@ private void query( private long getQueryTimeLowerBound(String[] device) { long deviceTTL = DataNodeTTLCache.getInstance().getTTL(device); - return deviceTTL != Long.MAX_VALUE - ? CommonDateTimeUtils.currentTime() - deviceTTL - : Long.MIN_VALUE; + return deviceTTL != Long.MAX_VALUE ? CommonUtils.getTTLLowerBound(deviceTTL) : Long.MIN_VALUE; } public long getTimeRangeId() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 9c1320503a31..c155404ee5e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.consensus.index.ProgressIndexType; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -40,6 +39,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.TimeIndexLevel; import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; +import org.apache.iotdb.db.utils.CommonUtils; import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -867,7 +867,7 @@ public boolean isSatisfied(IDeviceID deviceId, Filter timeFilter, boolean isSeq, * @return whether the given time falls in ttl */ private boolean isAlive(long time, long dataTTL) { - return dataTTL == Long.MAX_VALUE || (CommonDateTimeUtils.currentTime() - time) <= dataTTL; + return dataTTL == Long.MAX_VALUE || time >= CommonUtils.getTTLLowerBound(dataTTL); } /** @@ -1101,15 +1101,11 @@ public static int compareFileName(TsFileResource o1, TsFileResource o2) { public static int checkAndCompareFileName(String fileName1, String fileName2) throws IOException { TsFileNameGenerator.TsFileName tsFileName1 = TsFileNameGenerator.getTsFileName(fileName1); TsFileNameGenerator.TsFileName tsFileName2 = TsFileNameGenerator.getTsFileName(fileName2); - long timeDiff = tsFileName1.getTime() - tsFileName2.getTime(); - if (timeDiff != 0) { - return timeDiff < 0 ? -1 : 1; + int timeCompare = Long.compare(tsFileName1.getTime(), tsFileName2.getTime()); + if (timeCompare != 0) { + return timeCompare; } - long versionDiff = tsFileName1.getVersion() - tsFileName2.getVersion(); - if (versionDiff != 0) { - return versionDiff < 0 ? -1 : 1; - } - return 0; + return Long.compare(tsFileName1.getVersion(), tsFileName2.getVersion()); } /** @@ -1130,11 +1126,7 @@ public static int compareFileCreationOrderByDesc(TsFileResource o1, TsFileResour TsFileNameGenerator.getTsFileName(o1.getTsFile().getName()); TsFileNameGenerator.TsFileName n2 = TsFileNameGenerator.getTsFileName(o2.getTsFile().getName()); - long versionDiff = n2.getVersion() - n1.getVersion(); - if (versionDiff != 0) { - return versionDiff < 0 ? -1 : 1; - } - return 0; + return Long.compare(n2.getVersion(), n1.getVersion()); } catch (IOException e) { LOGGER.error("File name may not meet the standard naming specifications.", e); throw new RuntimeException(e.getMessage()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java index ea4088835536..a41cb3d4484e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java @@ -398,7 +398,8 @@ public static TsFileResource getSettleCompactionTargetFileResources( } public static class TsFileName { - private static final String FILE_NAME_PATTERN = "(\\d+)-(\\d+)-(\\d+)-(\\d+).tsfile$"; + // Timestamps may be negative for pre-epoch data, while the version is always non-negative. + private static final String FILE_NAME_PATTERN = "(-?\\d+)-(\\d+)-(\\d+)-(\\d+)\\.tsfile$"; private static final Pattern FILE_NAME_MATCHER = Pattern.compile(TsFileName.FILE_NAME_PATTERN); private long time; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java index 2e492647073b..58cd68226f30 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java @@ -21,11 +21,11 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.load.PartitionViolationException; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.CommonUtils; import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.file.metadata.IDeviceID; @@ -421,7 +421,7 @@ public boolean definitelyNotContains(IDeviceID device) { @Override public boolean isDeviceAlive(IDeviceID device, long ttl) { return ttl == Long.MAX_VALUE - || endTimes[deviceToIndex.get(device)] >= CommonDateTimeUtils.currentTime() - ttl; + || endTimes[deviceToIndex.get(device)] >= CommonUtils.getTTLLowerBound(ttl); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java index 72cd3d1c7977..4d423d165fd0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java @@ -20,11 +20,11 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.IOUtils; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.load.PartitionViolationException; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.CommonUtils; import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.file.metadata.IDeviceID; @@ -266,7 +266,7 @@ public boolean definitelyNotContains(IDeviceID device) { @Override public boolean isDeviceAlive(IDeviceID device, long ttl) { - return ttl == Long.MAX_VALUE || endTime >= CommonDateTimeUtils.currentTime() - ttl; + return ttl == Long.MAX_VALUE || endTime >= CommonUtils.getTTLLowerBound(ttl); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java index 1395eaa420dc..8343081d835e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java @@ -211,11 +211,7 @@ public void writeDecodePage(final long[] times, final Object[] values, final int pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); satisfiedLengthQueue.offer(satisfiedLength); final long startTime = timePartitionSlot.getStartTime(); - // beware of overflow - long endTime = startTime + TimePartitionUtils.getTimePartitionInterval() - 1; - if (endTime <= startTime) { - endTime = Long.MAX_VALUE; - } + final long endTime = TimePartitionUtils.getTimePartitionEndTime(startTime); // serialize needDecode==true dataSize += ReadWriteIOUtils.write(true, stream); dataSize += ReadWriteIOUtils.write(satisfiedLength, stream); @@ -235,11 +231,7 @@ public void writeDecodeValuePage( throws IOException { pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); final long startTime = timePartitionSlot.getStartTime(); - // beware of overflow - long endTime = startTime + TimePartitionUtils.getTimePartitionInterval() - 1; - if (endTime <= startTime) { - endTime = Long.MAX_VALUE; - } + final long endTime = TimePartitionUtils.getTimePartitionEndTime(startTime); final int satisfiedLength = satisfiedLengthQueue.poll(); // serialize needDecode==true dataSize += ReadWriteIOUtils.write(true, stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java index c34659399b52..5b8dd9d68459 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java @@ -63,11 +63,7 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT throws IOException { pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); final long startTime = timePartitionSlot.getStartTime(); - // beware of overflow - long endTime = startTime + TimePartitionUtils.getTimePartitionInterval() - 1; - if (endTime <= startTime) { - endTime = Long.MAX_VALUE; - } + final long endTime = TimePartitionUtils.getTimePartitionEndTime(startTime); final int satisfiedLength = satisfiedLengthQueue.poll(); // serialize needDecode==true dataSize += ReadWriteIOUtils.write(true, stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java index 6c5504e7a99d..2eb366f7b847 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java @@ -175,11 +175,7 @@ public void writeDecodePage(final long[] times, final Object[] values, final int throws IOException { pageNumber += 1; final long startTime = timePartitionSlot.getStartTime(); - // beware of overflow - long endTime = startTime + TimePartitionUtils.getTimePartitionInterval() - 1; - if (endTime <= startTime) { - endTime = Long.MAX_VALUE; - } + final long endTime = TimePartitionUtils.getTimePartitionEndTime(startTime); dataSize += ReadWriteIOUtils.write(true, stream); dataSize += ReadWriteIOUtils.write(satisfiedLength, stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java index 4571bdb1531d..549a857cee3e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java @@ -59,6 +59,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -73,7 +74,7 @@ public class TsFileSplitter { private final TsFileDataConsumer consumer; private Map offset2ChunkMetadata = new HashMap<>(); private TreeMap> offset2Deletions = new TreeMap<>(); - private Map> pageIndex2ChunkData = new HashMap<>(); + private Map> pageIndex2ChunkData = new LinkedHashMap<>(); private Map pageIndex2Times = new HashMap<>(); private boolean isTimeChunkNeedDecode = true; private IDeviceID curDevice = null; @@ -259,13 +260,10 @@ private void decodeAndWriteTimeChunkOrNonAlignedChunk( int satisfiedLength = 0; long endTime = - timePartitionSlot.getStartTime() + TimePartitionUtils.getTimePartitionInterval(); - // beware of overflow - if (endTime <= timePartitionSlot.getStartTime()) { - endTime = Long.MAX_VALUE; - } + TimePartitionUtils.getTimePartitionUpperBound(timePartitionSlot.getStartTime()); for (int i = 0; i < times.length; i++) { - if (times[i] >= endTime) { + if (TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + times[i], timePartitionSlot.getStartTime(), endTime)) { chunkData.writeDecodePage(times, values, satisfiedLength); if (isAligned) { pageIndex2ChunkData @@ -278,10 +276,7 @@ private void decodeAndWriteTimeChunkOrNonAlignedChunk( timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[i]); satisfiedLength = 0; endTime = - timePartitionSlot.getStartTime() + TimePartitionUtils.getTimePartitionInterval(); - if (endTime <= timePartitionSlot.getStartTime()) { - endTime = Long.MAX_VALUE; - } + TimePartitionUtils.getTimePartitionUpperBound(timePartitionSlot.getStartTime()); chunkData = ChunkData.createChunkData( isAligned, ((PlainDeviceID) curDevice).toStringID(), header, timePartitionSlot); @@ -368,7 +363,7 @@ private void storeTimeChunkContext() { pageIndex2ChunkDataList.add(pageIndex2ChunkData); isTimeChunkNeedDecodeList.add(isTimeChunkNeedDecode); pageIndex2Times = new HashMap<>(); - pageIndex2ChunkData = new HashMap<>(); + pageIndex2ChunkData = new LinkedHashMap<>(); isTimeChunkNeedDecode = true; } @@ -447,7 +442,7 @@ private void consumeAllAlignedChunkData( return; } - Map chunkDataMap = new HashMap<>(); + Map chunkDataMap = new LinkedHashMap<>(); for (Map.Entry> entry : pageIndex2ChunkData.entrySet()) { List alignedChunkDataList = entry.getValue(); for (int i = 0; i < alignedChunkDataList.size(); i++) { @@ -473,7 +468,7 @@ private void consumeAllAlignedChunkData( offset, chunkData)); } } - this.pageIndex2ChunkData = new HashMap<>(); + this.pageIndex2ChunkData = new LinkedHashMap<>(); } private void consumeChunkData(String measurement, long offset, ChunkData chunkData) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java index c008d174662d..6ef834258856 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java @@ -123,8 +123,17 @@ private static void moveSeqResourceToUnsequenceDir(TsFileResource resource) thro tsFileName.getInnerCompactionCnt(), 0); targetFile = new File(targetDir.getAbsolutePath() + File.separator + fileNameStr); + if (!targetFile.exists()) { + break; + } + if (tsFileName.getTime() == Long.MAX_VALUE) { + throw new IOException( + String.format( + "Cannot repair %s because the target file already exists and the file timestamp is Long.MAX_VALUE", + tsfile.getAbsolutePath())); + } tsFileName.setTime(tsFileName.getTime() + 1); - } while (targetFile.exists()); + } while (true); moveFile(tsfile, targetFile); moveFile( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java index 261ca957492e..0335df70946a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.thrift.OperationType; @@ -401,6 +402,30 @@ public static byte[] parseBlobStringToByteArray(String input) throws IllegalArgu } } + /** + * Check whether the time falls in TTL. + * + * @return whether the given time falls in ttl + */ + public static boolean isAlive(long time, long dataTTL) { + return dataTTL == Long.MAX_VALUE || time >= getTTLLowerBound(dataTTL); + } + + public static long getTTLLowerBound(long dataTTL) { + if (dataTTL == Long.MAX_VALUE) { + return Long.MIN_VALUE; + } + + long currentTime = CommonDateTimeUtils.currentTime(); + if (dataTTL >= 0 && currentTime < Long.MIN_VALUE + dataTTL) { + return Long.MIN_VALUE; + } + if (dataTTL < 0 && currentTime > Long.MAX_VALUE + dataTTL) { + return Long.MAX_VALUE; + } + return currentTime - dataTTL; + } + private static void badUse(Exception e) { System.out.println("node-tool: " + e.getMessage()); System.out.println("See 'node-tool help' or 'node-tool help '."); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java index a63eea9fe44d..f4cfa58e2b74 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java @@ -20,6 +20,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser; import org.apache.iotdb.db.qp.sql.SqlLexer; @@ -591,32 +592,43 @@ public static long convertDurationStrToLong( */ public static long convertDurationStrToLong( long currentTime, String duration, String timestampPrecision, boolean convertYearToMonth) { - long total = 0; - long temp = 0; - for (int i = 0; i < duration.length(); i++) { - char ch = duration.charAt(i); - if (Character.isDigit(ch)) { - temp *= 10; - temp += (ch - '0'); - } else { - String unit = String.valueOf(duration.charAt(i)); - // This is to identify units with two letters. - if (i + 1 < duration.length() && !Character.isDigit(duration.charAt(i + 1))) { - i++; - unit += duration.charAt(i); - } - unit = unit.toLowerCase(); - if (convertYearToMonth && unit.equals("y")) { - temp *= 12; - unit = "mo"; + try { + long total = 0; + long temp = 0; + for (int i = 0; i < duration.length(); i++) { + char ch = duration.charAt(i); + if (Character.isDigit(ch)) { + temp = Math.addExact(Math.multiplyExact(temp, 10), Character.digit(ch, 10)); + } else { + String unit = String.valueOf(duration.charAt(i)); + // This is to identify units with two letters. + if (i + 1 < duration.length() && !Character.isDigit(duration.charAt(i + 1))) { + i++; + unit += duration.charAt(i); + } + unit = unit.toLowerCase(); + if (convertYearToMonth && unit.equals("y")) { + temp = Math.multiplyExact(temp, 12); + unit = "mo"; + } + long componentCurrentTime = + isMonthUnit(unit) ? (currentTime == -1 ? -1 : Math.addExact(currentTime, total)) : -1; + total = + Math.addExact( + total, + DateTimeUtils.convertDurationStrToLong( + componentCurrentTime, temp, unit, timestampPrecision)); + temp = 0; } - total += - DateTimeUtils.convertDurationStrToLong( - currentTime == -1 ? -1 : currentTime + total, temp, unit, timestampPrecision); - temp = 0; } + return total; + } catch (ArithmeticException e) { + throw new SemanticException("Time duration is out of range."); } - return total; + } + + private static boolean isMonthUnit(String unit) { + return "mo".equals(unit) || "month".equals(unit); } @TestOnly @@ -628,37 +640,46 @@ public static long convertDurationStrToLongForTest( /** convert duration string to millisecond, microsecond or nanosecond. */ public static long convertDurationStrToLong( long currentTime, long value, String unit, String timestampPrecision) { + try { + return convertDurationStrToLongInternal(currentTime, value, unit, timestampPrecision); + } catch (ArithmeticException e) { + throw new SemanticException("Time duration is out of range."); + } + } + + private static long convertDurationStrToLongInternal( + long currentTime, long value, String unit, String timestampPrecision) { DurationUnit durationUnit = DurationUnit.valueOf(unit); long res = value; switch (durationUnit) { case y: - res *= 365 * 86_400_000L; + res = Math.multiplyExact(value, 365 * 86_400_000L); break; case mo: if (currentTime == -1) { - res *= 30 * 86_400_000L; + res = Math.multiplyExact(value, 30 * 86_400_000L); } else { Calendar calendar = Calendar.getInstance(); calendar.setTimeZone(SessionManager.getInstance().getSessionTimeZone()); calendar.setTimeInMillis(currentTime); - calendar.add(Calendar.MONTH, (int) (value)); - res = calendar.getTimeInMillis() - currentTime; + calendar.add(Calendar.MONTH, Math.toIntExact(value)); + res = Math.subtractExact(calendar.getTimeInMillis(), currentTime); } break; case w: - res *= 7 * 86_400_000L; + res = Math.multiplyExact(value, 7 * 86_400_000L); break; case d: - res *= 86_400_000L; + res = Math.multiplyExact(value, 86_400_000L); break; case h: - res *= 3_600_000L; + res = Math.multiplyExact(value, 3_600_000L); break; case m: - res *= 60_000L; + res = Math.multiplyExact(value, 60_000L); break; case s: - res *= 1_000L; + res = Math.multiplyExact(value, 1_000L); break; default: break; @@ -670,15 +691,15 @@ public static long convertDurationStrToLong( } else if (unit.equals(DurationUnit.us.toString())) { return value; } else { - return res * 1000; + return Math.multiplyExact(res, 1000); } } else if ("ns".equals(timestampPrecision)) { if (unit.equals(DurationUnit.ns.toString())) { return value; } else if (unit.equals(DurationUnit.us.toString())) { - return value * 1000; + return Math.multiplyExact(value, 1000); } else { - return res * 1000_000; + return Math.multiplyExact(res, 1000_000); } } else { if (unit.equals(DurationUnit.ns.toString())) { @@ -779,39 +800,44 @@ public static long calcPositiveIntervalByMonth( * @return the TimeDuration instance contains month part and non-month part */ public static TimeDuration constructTimeDuration(String duration) { - duration = duration.toLowerCase(); - String currTimePrecision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); - long temp = 0; - long monthDuration = 0; - long nonMonthDuration = 0; - for (int i = 0; i < duration.length(); i++) { - char ch = duration.charAt(i); - if (Character.isDigit(ch)) { - temp *= 10; - temp += (ch - '0'); - } else { - String unit = String.valueOf(duration.charAt(i)); - // This is to identify units with two letters. - if (i + 1 < duration.length() && !Character.isDigit(duration.charAt(i + 1))) { - i++; - unit += duration.charAt(i); - } - if (unit.equals("y")) { - monthDuration += temp * 12; - temp = 0; - continue; - } - if (unit.equals("mo")) { - monthDuration += temp; + try { + duration = duration.toLowerCase(); + String currTimePrecision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + long temp = 0; + long monthDuration = 0; + long nonMonthDuration = 0; + for (int i = 0; i < duration.length(); i++) { + char ch = duration.charAt(i); + if (Character.isDigit(ch)) { + temp = Math.addExact(Math.multiplyExact(temp, 10), Character.digit(ch, 10)); + } else { + String unit = String.valueOf(duration.charAt(i)); + // This is to identify units with two letters. + if (i + 1 < duration.length() && !Character.isDigit(duration.charAt(i + 1))) { + i++; + unit += duration.charAt(i); + } + if (unit.equals("y")) { + monthDuration = Math.addExact(monthDuration, Math.multiplyExact(temp, 12)); + temp = 0; + continue; + } + if (unit.equals("mo")) { + monthDuration = Math.addExact(monthDuration, temp); + temp = 0; + continue; + } + nonMonthDuration = + Math.addExact( + nonMonthDuration, + DateTimeUtils.convertDurationStrToLong(-1, temp, unit, currTimePrecision)); temp = 0; - continue; } - nonMonthDuration += - DateTimeUtils.convertDurationStrToLong(-1, temp, unit, currTimePrecision); - temp = 0; } + return new TimeDuration(Math.toIntExact(monthDuration), nonMonthDuration); + } catch (ArithmeticException e) { + throw new SemanticException("Time duration is out of range."); } - return new TimeDuration((int) monthDuration, nonMonthDuration); } public static Long parseDateTimeExpressionToLong(String dateExpression, ZoneId zoneId) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java index 4df2cfbc459c..43016d871ed0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.utils; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; - import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.filter.basic.Filter; @@ -40,7 +38,7 @@ public TimeFilterForDeviceTTL(Filter timeFilter, Map ttlCached) public boolean satisfyStartEndTime(long startTime, long endTime, IDeviceID deviceID) { long ttl = getTTL(deviceID); if (ttl != Long.MAX_VALUE) { - long validStartTime = CommonDateTimeUtils.currentTime() - ttl; + long validStartTime = CommonUtils.getTTLLowerBound(ttl); if (validStartTime > endTime) { return false; } @@ -52,7 +50,7 @@ public boolean satisfyStartEndTime(long startTime, long endTime, IDeviceID devic public boolean satisfy(long time, IDeviceID deviceID) { long ttl = getTTL(deviceID); if (ttl != Long.MAX_VALUE) { - long validStartTime = CommonDateTimeUtils.currentTime() - ttl; + long validStartTime = CommonUtils.getTTLLowerBound(ttl); if (validStartTime > time) { return false; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/handler/SlidingTimeWindowEvaluationHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/handler/SlidingTimeWindowEvaluationHandler.java index 92c3044ff40f..2baaffb6d114 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/handler/SlidingTimeWindowEvaluationHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/handler/SlidingTimeWindowEvaluationHandler.java @@ -28,6 +28,8 @@ import java.util.LinkedList; import java.util.Queue; +import static com.google.common.math.LongMath.saturatedAdd; + public class SlidingTimeWindowEvaluationHandler extends SlidingWindowEvaluationHandler { private final long timeInterval; @@ -38,9 +40,13 @@ public class SlidingTimeWindowEvaluationHandler extends SlidingWindowEvaluationH /** window: [begin, end). */ private long currentWindowEndTime; + private boolean currentWindowEndTimeOverflow; + /** window: [begin, end). */ private long nextWindowBeginTime; + private boolean nextWindowBeginTimeOverflow; + public SlidingTimeWindowEvaluationHandler( SlidingTimeWindowConfiguration configuration, Evaluator evaluator) throws WindowingException { super(configuration, evaluator); @@ -55,24 +61,28 @@ public SlidingTimeWindowEvaluationHandler( protected void createEvaluationTaskIfNecessary(long timestamp) { if (data.size() == 1) { windowBeginIndexQueue.add(0); - currentWindowEndTime = timestamp + timeInterval; - nextWindowBeginTime = timestamp + slidingStep; + currentWindowEndTime = saturatedAdd(timestamp, timeInterval); + currentWindowEndTimeOverflow = timestamp > Long.MAX_VALUE - timeInterval; + nextWindowBeginTime = saturatedAdd(timestamp, slidingStep); + nextWindowBeginTimeOverflow = timestamp > Long.MAX_VALUE - slidingStep; return; } - while (nextWindowBeginTime <= timestamp) { + while (!nextWindowBeginTimeOverflow && nextWindowBeginTime <= timestamp) { windowBeginIndexQueue.add(data.size() - 1); - nextWindowBeginTime += slidingStep; + nextWindowBeginTimeOverflow = nextWindowBeginTime > Long.MAX_VALUE - slidingStep; + nextWindowBeginTime = saturatedAdd(nextWindowBeginTime, slidingStep); } - while (currentWindowEndTime <= timestamp) { + while (!currentWindowEndTimeOverflow && currentWindowEndTime <= timestamp) { int windowBeginIndex = windowBeginIndexQueue.remove(); TASK_POOL_MANAGER.submit( new WindowEvaluationTask( evaluator, new WindowImpl(data, windowBeginIndex, data.size() - 1 - windowBeginIndex))); data.setEvictionUpperBound(windowBeginIndex); - currentWindowEndTime += slidingStep; + currentWindowEndTimeOverflow = currentWindowEndTime > Long.MAX_VALUE - slidingStep; + currentWindowEndTime = saturatedAdd(currentWindowEndTime, slidingStep); } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessorTest.java new file mode 100644 index 000000000000..7d4e8d07cf00 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessorTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.processor.aggregate.window.processor; + +import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.TimeSeriesWindow; +import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowOutput; +import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowState; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + +import org.apache.tsfile.utils.Pair; +import org.junit.Assert; +import org.junit.Test; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_SLIDING_BOUNDARY_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_SLIDING_SECONDS_KEY; + +public class TumblingWindowingProcessorTest { + + @Test + public void testMayAddWindowAlignsExtremeRangeWithoutOverflow() throws Exception { + final TumblingWindowingProcessor processor = createProcessor(Long.MIN_VALUE, 1); + final long interval = TimestampPrecisionUtils.convertToCurrPrecision(1, TimeUnit.SECONDS); + final List windows = new ArrayList<>(); + + Assert.assertEquals(1, processor.mayAddWindow(windows, Long.MAX_VALUE).size()); + Assert.assertEquals(1, windows.size()); + Assert.assertEquals( + alignWindowStart(Long.MAX_VALUE, Long.MIN_VALUE, interval), windows.get(0).getTimestamp()); + Assert.assertTrue(windows.get(0).getTimestamp() > Long.MIN_VALUE); + } + + @Test + public void testWindowEndOverflowDoesNotEmitEarly() throws Exception { + final TumblingWindowingProcessor processor = createProcessor(0, 1); + final long interval = TimestampPrecisionUtils.convertToCurrPrecision(1, TimeUnit.SECONDS); + final TimeSeriesWindow window = new TimeSeriesWindow(processor, null); + window.setTimestamp(Long.MAX_VALUE - interval + 1); + + final Pair result = + processor.updateAndMaySetWindowState(window, Long.MAX_VALUE); + + Assert.assertEquals(WindowState.COMPUTE, result.getLeft()); + Assert.assertNull(result.getRight()); + Assert.assertEquals(Long.MAX_VALUE, processor.forceOutput(window).getProgressTime()); + + final List windows = new ArrayList<>(); + windows.add(window); + Assert.assertTrue(processor.mayAddWindow(windows, Long.MAX_VALUE).isEmpty()); + } + + private static TumblingWindowingProcessor createProcessor( + final long slidingBoundaryTime, final long slidingSeconds) throws Exception { + final Map attributes = new HashMap<>(); + attributes.put(PROCESSOR_SLIDING_BOUNDARY_TIME_KEY, Long.toString(slidingBoundaryTime)); + attributes.put(PROCESSOR_SLIDING_SECONDS_KEY, Long.toString(slidingSeconds)); + final PipeParameters parameters = new PipeParameters(attributes); + final TumblingWindowingProcessor processor = new TumblingWindowingProcessor(); + processor.validate(new PipeParameterValidator(parameters)); + processor.customize(parameters, () -> null); + return processor; + } + + private static long alignWindowStart( + final long timestamp, final long baseTime, final long interval) { + final BigInteger base = BigInteger.valueOf(baseTime); + final BigInteger intervalValue = BigInteger.valueOf(interval); + return base.add( + BigInteger.valueOf(timestamp) + .subtract(base) + .divide(intervalValue) + .multiply(intervalValue)) + .longValueExact(); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjectorTest.java new file mode 100644 index 000000000000..6141cbc7196e --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjectorTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion; + +import org.junit.Assert; +import org.junit.Test; + +public class DataRegionWatermarkInjectorTest { + + @Test + public void testCalculateNextInjectionTime() { + Assert.assertEquals( + 90_000, DataRegionWatermarkInjector.calculateNextInjectionTime(60_001, 30_000)); + } + + @Test + public void testCalculateNextInjectionTimeSaturatesOnOverflow() { + Assert.assertEquals( + Long.MAX_VALUE, + DataRegionWatermarkInjector.calculateNextInjectionTime( + Long.MAX_VALUE, DataRegionWatermarkInjector.MIN_INJECTION_INTERVAL_IN_MS)); + Assert.assertEquals( + Long.MAX_VALUE, + DataRegionWatermarkInjector.calculateNextInjectionTime( + Long.MAX_VALUE - 1, DataRegionWatermarkInjector.MIN_INJECTION_INTERVAL_IN_MS)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorTest.java index 45ea1ddc9cdc..99adaf5da658 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorTest.java @@ -441,6 +441,25 @@ public void minTimeAccumulatorTest() { Assert.assertEquals(100, finalResult.build().getLong(0)); } + @Test + public void timeDurationAccumulatorSaturatesOverflow() { + TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Collections.singletonList(TSDataType.INT32)); + tsBlockBuilder.getTimeColumnBuilder().writeLong(Long.MIN_VALUE); + tsBlockBuilder.getColumnBuilder(0).writeInt(1); + tsBlockBuilder.declarePosition(); + tsBlockBuilder.getTimeColumnBuilder().writeLong(Long.MAX_VALUE); + tsBlockBuilder.getColumnBuilder(0).writeInt(2); + tsBlockBuilder.declarePosition(); + TsBlock tsBlock = tsBlockBuilder.build(); + + TimeDurationAccumulator accumulator = new TimeDurationAccumulator(); + accumulator.addInput(new Column[] {tsBlock.getTimeColumn(), tsBlock.getColumn(0)}, null); + + ColumnBuilder finalResult = new LongColumnBuilder(null, 1); + accumulator.outputFinal(finalResult); + Assert.assertEquals(Long.MAX_VALUE, finalResult.build().getLong(0)); + } + @Test public void maxValueAccumulatorTest() { Accumulator extremeAccumulator = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeRangeIteratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeRangeIteratorTest.java index 86d8ef8ad34b..c8f71367c930 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeRangeIteratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeRangeIteratorTest.java @@ -66,6 +66,133 @@ public void testNotSplitTimeRange() { checkRes(descTimeRangeIterator, res); } + @Test + public void testNotSplitTimeRangeWithLongBoundaries() { + TimeDuration interval = new TimeDuration(0, 2); + TimeDuration slidingStep = new TimeDuration(0, 1); + + String[] maxLeftClosedRightOpenRes = { + "[ 9223372036854775805 : 9223372036854775806 ]", + "[ 9223372036854775806 : 9223372036854775806 ]" + }; + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MAX_VALUE - 2, + Long.MAX_VALUE, + interval, + slidingStep, + true, + true, + false, + ZoneId.systemDefault()), + maxLeftClosedRightOpenRes); + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MAX_VALUE - 2, + Long.MAX_VALUE, + interval, + slidingStep, + false, + true, + false, + ZoneId.systemDefault()), + maxLeftClosedRightOpenRes); + + String[] maxLeftOpenRightClosedRes = { + "[ 9223372036854775806 : 9223372036854775807 ]", + "[ 9223372036854775807 : 9223372036854775807 ]" + }; + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MAX_VALUE - 2, + Long.MAX_VALUE, + interval, + slidingStep, + true, + false, + false, + ZoneId.systemDefault()), + maxLeftOpenRightClosedRes); + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MAX_VALUE - 2, + Long.MAX_VALUE, + interval, + slidingStep, + false, + false, + false, + ZoneId.systemDefault()), + maxLeftOpenRightClosedRes); + + String[] minLeftClosedRightOpenRes = { + "[ -9223372036854775808 : -9223372036854775807 ]", + "[ -9223372036854775807 : -9223372036854775807 ]" + }; + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MIN_VALUE, + Long.MIN_VALUE + 2, + interval, + slidingStep, + true, + true, + false, + ZoneId.systemDefault()), + minLeftClosedRightOpenRes); + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MIN_VALUE, + Long.MIN_VALUE + 2, + interval, + slidingStep, + false, + true, + false, + ZoneId.systemDefault()), + minLeftClosedRightOpenRes); + + String[] maxSplitLeftClosedRightOpenRes = { + "[ 9223372036854775805 : 9223372036854775805 ]", + "[ 9223372036854775806 : 9223372036854775806 ]" + }; + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MAX_VALUE - 2, + Long.MAX_VALUE, + interval, + slidingStep, + true, + true, + true, + ZoneId.systemDefault()), + maxSplitLeftClosedRightOpenRes); + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MAX_VALUE - 2, + Long.MAX_VALUE, + interval, + slidingStep, + false, + true, + true, + ZoneId.systemDefault()), + maxSplitLeftClosedRightOpenRes); + + Assert.assertEquals( + Long.MAX_VALUE, + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MIN_VALUE, + Long.MAX_VALUE, + new TimeDuration(0, 1), + new TimeDuration(0, 1), + true, + true, + false, + ZoneId.systemDefault()) + .getTotalIntervalNum()); + } + @Test public void testSplitTimeRange() { String[] res4_1 = { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperatorTest.java new file mode 100644 index 000000000000..2ee0854c4b14 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperatorTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.ai; + +import org.apache.iotdb.db.exception.runtime.ModelInferenceProcessException; + +import org.junit.Assert; +import org.junit.Test; + +public class InferenceOperatorTest { + + @Test + public void testGeneratedTimeRejectsOverflow() { + Assert.assertEquals( + Long.MAX_VALUE, InferenceOperator.calculateGeneratedTime(Long.MAX_VALUE - 2, 1, 2)); + + Assert.assertThrows( + ModelInferenceProcessException.class, + () -> InferenceOperator.calculateGeneratedTime(Long.MAX_VALUE - 1, 1, 2)); + } + + @Test + public void testGeneratedTimeIntervalRejectsOverflow() { + Assert.assertEquals( + Long.MAX_VALUE, + InferenceOperator.calculateGeneratedTimeInterval(Long.MIN_VALUE, Long.MAX_VALUE, 2)); + + Assert.assertThrows( + ModelInferenceProcessException.class, + () -> InferenceOperator.calculateGeneratedTimeInterval(Long.MIN_VALUE, Long.MAX_VALUE, 1)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/ConvertPredicateToTimeFilterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/ConvertPredicateToTimeFilterTest.java index 3e8c75f943d9..895cd5321479 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/ConvertPredicateToTimeFilterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/ConvertPredicateToTimeFilterTest.java @@ -105,6 +105,11 @@ public void testNormal() { parameter.getSlidingStep(), TimeZone.getTimeZone("+00:00"), TimestampPrecisionUtils.currPrecision)); + + GroupByTimeParameter rightClosedParameter = + new GroupByTimeParameter( + 1, Long.MAX_VALUE, new TimeDuration(0, 10), new TimeDuration(0, 10), false); + Assert.assertThrows(IllegalArgumentException.class, () -> groupByTime(rightClosedParameter)); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeTest.java index f7b47615bd7e..228ed7866d86 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeTest.java @@ -68,10 +68,12 @@ import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.DEVICE; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.and; +import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.eq; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.groupByTime; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.gt; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.gte; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.longValue; +import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.or; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.time; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -274,6 +276,17 @@ public void testAggregationQuery() { } } + @Test + public void testRightClosedGroupByTimePredicateWithLongMaxEndTime() { + String sql = "select count(s1) from root.sg.d1 " + "group by ((0, 9223372036854775807], 10ms);"; + + Analysis actualAnalysis = analyzeSQL(sql); + + assertEquals( + or(groupByTime(1, Long.MAX_VALUE, 10, 10), eq(time(), longValue(Long.MAX_VALUE))), + actualAnalysis.getGlobalTimePredicate()); + } + @Test public void testRawDataQueryAlignByDevice() { String sql = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java index 1e85a1461ee8..aa31e0ed3762 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java @@ -20,6 +20,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.QueryId; @@ -575,4 +576,30 @@ public void testGetTimePartitionSlotList() { assertFalse(res.right.left); assertFalse(res.right.right); } + + @Test + public void testEstimateTimePartitionSlotMemoryWithOverflow() { + assertEquals(Long.MAX_VALUE, AnalyzeVisitor.estimateTimePartitionSlotMemory(Long.MAX_VALUE)); + } + + @Test + public void testTimePartitionSlotListWithUpperOverflowPartition() { + MPPQueryContext context = new MPPQueryContext(new QueryId("test_query")); + long partitionStartTime = TimePartitionUtils.getTimePartitionSlot(Long.MAX_VALUE).startTime; + + Pair, Pair> res = + getTimePartitionSlotList( + TimeFilterApi.between(partitionStartTime - 1, Long.MAX_VALUE - 1), context); + List expected = + Arrays.asList( + TimePartitionUtils.getTimePartitionSlot(partitionStartTime - 1), + new TTimePartitionSlot(partitionStartTime)); + + assertEquals(expected.size(), res.left.size()); + for (int i = 0; i < expected.size(); i++) { + assertEquals(expected.get(i), res.left.get(i)); + } + assertFalse(res.right.left); + assertFalse(res.right.right); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionSelectorTest.java index 88cd04bb8e64..e02398416dcd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionSelectorTest.java @@ -35,6 +35,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.datastructure.FixedPriorityBlockingQueue; import org.apache.tsfile.exception.write.WriteProcessException; @@ -271,7 +272,7 @@ public void testSeqFileWithDeviceIndexBeenDeletedBeforeSelection() "", "", 0, null, new CompactionScheduleContext()); CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -342,7 +343,7 @@ public void testSeqFileWithDeviceIndexBeenDeletedDuringSelectionAndAfterCopyingL // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); // the other thread holds write lock and delete file successfully after copying list cd1.countDown(); @@ -405,7 +406,7 @@ public void testSeqFileWithDeviceIndexBeenDeletedDuringSelectionAndBeforeSetting // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -511,7 +512,7 @@ public void testSeqFileWithDeviceIndexBeenDeletedDuringSelectionAndBeforeSetting // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -632,7 +633,7 @@ public void testSeqFileWithFileIndexBeenDeletedDuringSelectionAndAfterCopyingLis // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); // the other thread holds write lock and delete file successfully after copying list cd1.countDown(); @@ -749,7 +750,7 @@ public void testSeqFileWithFileIndexBeenDeletedDuringSelectionAndBeforeSettingCa // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -859,7 +860,7 @@ public void testSeqFileWithFileIndexBeenDeletedDuringSelectionAndBeforeSettingCo // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -981,7 +982,7 @@ public void testSeqFileWithFileIndexBeenDeletedBeforeSelection() "", "", 0, null, new CompactionScheduleContext()); CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -1053,7 +1054,7 @@ public void testUnSeqFileWithDeviceIndexBeenDeletedBeforeSelection() "", "", 0, null, new CompactionScheduleContext()); CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -1124,7 +1125,7 @@ public void testUnSeqFileWithDeviceIndexBeenDeletedDuringSelectionAndAfterCopyin // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); // the other thread holds write lock and delete file successfully after copying list cd1.countDown(); @@ -1188,7 +1189,7 @@ public void testUnSeqFileWithDeviceIndexBeenDeletedDuringSelectionAndBeforeSetti // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -1295,7 +1296,7 @@ public void testUnSeqFileWithDeviceIndexBeenDeletedDuringSelectionAndBeforeSetti // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -1417,7 +1418,7 @@ public void testUnSeqFileWithFileIndexBeenDeletedDuringSelectionAndAfterCopyingL // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); // the other thread holds write lock and delete file successfully after copying list cd1.countDown(); @@ -1533,7 +1534,7 @@ public void testUnSeqFileWithFileIndexBeenDeletedDuringSelectionAndBeforeSetting // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -1643,7 +1644,7 @@ public void testUnSeqFileWithFileIndexBeenDeletedDuringSelectionAndBeforeSetting // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -1765,7 +1766,7 @@ public void testUnSeqFileWithFileIndexBeenDeletedBeforeSelection() "", "", 0, null, new CompactionScheduleContext()); CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java index e327114ca5c9..d3ae001b2154 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java @@ -43,6 +43,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceList; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.tsfile.read.TimeValuePair; @@ -407,7 +408,7 @@ public void testOneSeqFileAndSixUnseqFile() throws Exception { seqTsFileResourceList.addAll(seqResources); TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); - long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long timeLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); @@ -709,7 +710,7 @@ public void testFiveSeqFileAndOneUnseqFileWithSomeDeviceNotInSeqFiles() throws E seqTsFileResourceList.addAll(seqResources); TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); - long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long timeLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); @@ -1009,7 +1010,7 @@ public void testFiveSeqFileAndOneUnseqFile() throws Exception { seqTsFileResourceList.addAll(seqResources); TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); - long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long timeLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java index 3c194a4e3d14..9193c8232233 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java @@ -43,6 +43,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceList; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.tsfile.read.TimeValuePair; @@ -406,7 +407,7 @@ public void testOneSeqFileAndSixUnseqFile() throws Exception { seqTsFileResourceList.addAll(seqResources); TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); - long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long timeLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); @@ -708,7 +709,7 @@ public void testFiveSeqFileAndOneUnseqFileWithSomeDeviceNotInSeqFiles() throws E seqTsFileResourceList.addAll(seqResources); TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); - long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long timeLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); @@ -1008,7 +1009,7 @@ public void testFiveSeqFileAndOneUnseqFile() throws Exception { seqTsFileResourceList.addAll(seqResources); TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); - long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long timeLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java index 9b12bee9df87..98b014bdfaae 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.tsfile.exception.write.WriteProcessException; @@ -237,7 +238,7 @@ public void testFileOpenSelectionFromCompaction() List newUnseqResources = new ArrayList<>(); newUnseqResources.add(largeUnseqTsFileResource); - long ttlLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long ttlLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate(seqResources, newUnseqResources, ttlLowerBound); assertEquals(5, mergeResource.getSeqFiles().size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkTimeRangeUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkTimeRangeUtilsTest.java new file mode 100644 index 000000000000..b01fe53be947 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkTimeRangeUtilsTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.memtable; + +import org.junit.Test; + +import java.math.BigInteger; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class MemChunkTimeRangeUtilsTest { + + @Test + public void testSplitFakePageTimeRangesWithLongBoundaries() { + List pageTimeRanges = + MemChunkTimeRangeUtils.splitFakePageTimeRanges(Long.MIN_VALUE, Long.MAX_VALUE, 100); + + assertEquals(100, pageTimeRanges.size()); + assertEquals(Long.MIN_VALUE, pageTimeRanges.get(0)[0]); + assertEquals(Long.MAX_VALUE, pageTimeRanges.get(pageTimeRanges.size() - 1)[1]); + for (int i = 0; i < pageTimeRanges.size(); i++) { + long[] pageTimeRange = pageTimeRanges.get(i); + assertTrue(pageTimeRange[0] <= pageTimeRange[1]); + if (i > 0) { + assertEquals( + BigInteger.valueOf(pageTimeRanges.get(i - 1)[1]).add(BigInteger.ONE), + BigInteger.valueOf(pageTimeRange[0])); + } + } + } + + @Test + public void testSplitFakePageTimeRangesDoesNotCreateEmptyRanges() { + List pageTimeRanges = MemChunkTimeRangeUtils.splitFakePageTimeRanges(10, 10, 100); + + assertEquals(1, pageTimeRanges.size()); + assertArrayEquals(new long[] {10, 10}, pageTimeRanges.get(0)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java index 17da996f6c66..10e70292d6d1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java @@ -83,6 +83,36 @@ public void testSerializeAndDeserialize() throws IOException { Assert.assertEquals(tsFileResource, derTsFileResource); } + @Test + public void testCheckAndCompareFileNameWithLongBoundary() throws IOException { + String minTimeFile = TsFileNameGenerator.generateNewTsFileName(Long.MIN_VALUE, 0, 0, 0); + String maxTimeFile = TsFileNameGenerator.generateNewTsFileName(Long.MAX_VALUE, 0, 0, 0); + + Assert.assertTrue(TsFileResource.checkAndCompareFileName(minTimeFile, maxTimeFile) < 0); + Assert.assertTrue(TsFileResource.checkAndCompareFileName(maxTimeFile, minTimeFile) > 0); + + String minVersionFile = TsFileNameGenerator.generateNewTsFileName(0, Long.MAX_VALUE - 1, 0, 0); + String maxVersionFile = TsFileNameGenerator.generateNewTsFileName(0, Long.MAX_VALUE, 0, 0); + + Assert.assertTrue(TsFileResource.checkAndCompareFileName(minVersionFile, maxVersionFile) < 0); + Assert.assertTrue(TsFileResource.checkAndCompareFileName(maxVersionFile, minVersionFile) > 0); + } + + @Test + public void testCompareFileCreationOrderByDescWithLongBoundaryVersion() { + TsFileResource minVersionResource = + new TsFileResource( + new File(TsFileNameGenerator.generateNewTsFileName(0, Long.MAX_VALUE - 1, 0, 0))); + TsFileResource maxVersionResource = + new TsFileResource( + new File(TsFileNameGenerator.generateNewTsFileName(0, Long.MAX_VALUE, 0, 0))); + + Assert.assertTrue( + TsFileResource.compareFileCreationOrderByDesc(maxVersionResource, minVersionResource) < 0); + Assert.assertTrue( + TsFileResource.compareFileCreationOrderByDesc(minVersionResource, maxVersionResource) > 0); + } + @Test public void testSerializeDegradedTimeIndex() throws IOException { tsFileResource.serialize(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/CommonUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/CommonUtilsTest.java new file mode 100644 index 000000000000..bd1b6c52ea71 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/CommonUtilsTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import org.junit.Assert; +import org.junit.Test; + +public class CommonUtilsTest { + + @Test + public void testIsAliveDoesNotOverflowForLongMinTimestamp() { + Assert.assertFalse(CommonUtils.isAlive(Long.MIN_VALUE, 1)); + Assert.assertTrue(CommonUtils.isAlive(Long.MAX_VALUE, 1)); + Assert.assertTrue(CommonUtils.isAlive(Long.MIN_VALUE, Long.MAX_VALUE)); + } + + @Test + public void testTTLLowerBoundDoesNotUnderflowWithHugeTTL() { + long ttl = Long.MAX_VALUE - 1; + long ttlLowerBound = CommonUtils.getTTLLowerBound(ttl); + + Assert.assertTrue(ttlLowerBound > Long.MIN_VALUE); + Assert.assertFalse(CommonUtils.isAlive(Long.MIN_VALUE, ttl)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/windowing/SlidingTimeWindowEvaluationHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/windowing/SlidingTimeWindowEvaluationHandlerTest.java index 019aadde0f44..71110c8417b9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/windowing/SlidingTimeWindowEvaluationHandlerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/windowing/SlidingTimeWindowEvaluationHandlerTest.java @@ -149,6 +149,21 @@ public void test19() throws WindowingException { doTest(1, 100, 101); } + @Test(timeout = 1000) + public void testNearLongMaxBoundaryDoesNotOverflow() throws WindowingException { + final AtomicInteger count = new AtomicInteger(0); + + SlidingTimeWindowEvaluationHandler handler = + new SlidingTimeWindowEvaluationHandler( + new SlidingTimeWindowConfiguration(TSDataType.INT32, 2, 1), + window -> count.incrementAndGet()); + + handler.collect(Long.MAX_VALUE - 1, 1); + handler.collect(Long.MAX_VALUE, 2); + + Assert.assertEquals(0, count.get()); + } + private void doTest(long timeInterval, long slidingStep, long totalTime) throws WindowingException { final AtomicInteger count = new AtomicInteger(0); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java index da8952051e51..815043082ed3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java @@ -50,6 +50,8 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; +import static com.google.common.math.LongMath.saturatedAdd; + public class SeriesPartitionTable { // should only be used in CN scope, in DN scope should directly use @@ -186,16 +188,22 @@ public List getTimeSlotList( TConsensusGroupId regionId, long startTime, long endTime) { if (regionId.getId() == -1) { return seriesPartitionMap.keySet().stream() - .filter(e -> e.getStartTime() >= startTime && e.getStartTime() < endTime) + .filter(e -> isTimeSlotInQueryRange(e, startTime, endTime)) .collect(Collectors.toList()); } else { return seriesPartitionMap.keySet().stream() - .filter(e -> e.getStartTime() >= startTime && e.getStartTime() < endTime) + .filter(e -> isTimeSlotInQueryRange(e, startTime, endTime)) .filter(e -> seriesPartitionMap.get(e).contains(regionId)) .collect(Collectors.toList()); } } + private static boolean isTimeSlotInQueryRange( + TTimePartitionSlot timePartitionSlot, long startTime, long endTime) { + final long slotStartTime = timePartitionSlot.getStartTime(); + return slotStartTime >= startTime && (endTime == Long.MAX_VALUE || slotStartTime < endTime); + } + /** * Create DataPartition within the specific SeriesPartitionSlot. * @@ -269,8 +277,7 @@ public List autoCleanPartitionTable( while (iterator.hasNext()) { Map.Entry> entry = iterator.next(); TTimePartitionSlot timePartitionSlot = entry.getKey(); - if (timePartitionSlot.getStartTime() + TIME_PARTITION_INTERVAL + TTL - <= currentTimeSlot.getStartTime()) { + if (isTimePartitionExpired(timePartitionSlot, TTL, currentTimeSlot)) { removedTimePartitions.add(timePartitionSlot); iterator.remove(); } @@ -278,6 +285,13 @@ public List autoCleanPartitionTable( return removedTimePartitions; } + private static boolean isTimePartitionExpired( + TTimePartitionSlot timePartitionSlot, long TTL, TTimePartitionSlot currentTimeSlot) { + long partitionEndTime = saturatedAdd(timePartitionSlot.getStartTime(), TIME_PARTITION_INTERVAL); + long expireTime = saturatedAdd(partitionEndTime, TTL); + return expireTime <= currentTimeSlot.getStartTime(); + } + public void merge(SeriesPartitionTable sourceMap) { if (sourceMap == null) return; sourceMap.seriesPartitionMap.forEach( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java index f3b3aacba378..de66b83eb442 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java @@ -40,6 +40,8 @@ import java.util.Comparator; import java.util.PriorityQueue; +import static org.apache.iotdb.commons.utils.CommonDateTimeUtils.timeDifferenceAsDouble; + public class UDTFEqualSizeBucketOutlierSample extends UDTFEqualSizeBucketSample { private String type; @@ -184,13 +186,16 @@ public void outlierSampleInt(RowWindow rowWindow, PointCollector collector) thro new PriorityQueue<>(number, Comparator.comparing(o -> o.right)); double A = (double) row0y - row1y; - double B = (double) row1x - row0x; - double C = (double) row0x * row1y - row1x * row0y; + double B = timeDifferenceAsDouble(row1x, row0x); double denominator = Math.sqrt(A * A + B * B); for (int i = 1; i < windowSize - 1; i++) { Row row = rowWindow.getRow(i); - double value = Math.abs(A * row.getTime() + B * row.getInt(0) + C) / denominator; + double value = + Math.abs( + A * timeDifferenceAsDouble(row.getTime(), row0x) + + B * ((double) row.getInt(0) - row0y)) + / denominator; addToMinHeap(pq, i, value); } @@ -213,14 +218,17 @@ public void outlierSampleLong(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> o.right)); - double A = (double) row0y - row1y; - double B = (double) row1x - row0x; - double C = (double) row0x * row1y - row1x * row0y; + double A = timeDifferenceAsDouble(row0y, row1y); + double B = timeDifferenceAsDouble(row1x, row0x); double denominator = Math.sqrt(A * A + B * B); for (int i = 1; i < windowSize - 1; i++) { Row row = rowWindow.getRow(i); - double value = Math.abs(A * row.getTime() + B * row.getLong(0) + C) / denominator; + double value = + Math.abs( + A * timeDifferenceAsDouble(row.getTime(), row0x) + + B * timeDifferenceAsDouble(row.getLong(0), row0y)) + / denominator; addToMinHeap(pq, i, value); } @@ -244,13 +252,16 @@ public void outlierSampleFloat(RowWindow rowWindow, PointCollector collector) new PriorityQueue<>(number, Comparator.comparing(o -> o.right)); double A = (double) row0y - row1y; - double B = (double) row1x - row0x; - double C = (double) row0x * row1y - row1x * row0y; + double B = timeDifferenceAsDouble(row1x, row0x); double denominator = Math.sqrt(A * A + B * B); for (int i = 1; i < windowSize - 1; i++) { Row row = rowWindow.getRow(i); - double value = Math.abs(A * row.getTime() + B * row.getFloat(0) + C) / denominator; + double value = + Math.abs( + A * timeDifferenceAsDouble(row.getTime(), row0x) + + B * ((double) row.getFloat(0) - row0y)) + / denominator; addToMinHeap(pq, i, value); } @@ -274,13 +285,16 @@ public void outlierSampleDouble(RowWindow rowWindow, PointCollector collector) new PriorityQueue<>(number, Comparator.comparing(o -> o.right)); double A = row0y - row1y; - double B = (double) row1x - row0x; - double C = row0x * row1y - row1x * row0y; + double B = timeDifferenceAsDouble(row1x, row0x); double denominator = Math.sqrt(A * A + B * B); for (int i = 1; i < windowSize - 1; i++) { Row row = rowWindow.getRow(i); - double value = Math.abs(A * row.getTime() + B * row.getDouble(0) + C) / denominator; + double value = + Math.abs( + A * timeDifferenceAsDouble(row.getTime(), row0x) + + B * (row.getDouble(0) - row0y)) + / denominator; addToMinHeap(pq, i, value); } @@ -301,8 +315,9 @@ public void outlierSampleInt(RowWindow rowWindow, PointCollector collector) thro PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> -o.right)); - long lastTime, currentTime, nextTime, x1, x2; - int lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + int lastValue, currentValue, nextValue; + double x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -314,14 +329,12 @@ public void outlierSampleInt(RowWindow rowWindow, PointCollector collector) thro currentValue = rowWindow.getRow(i).getInt(0); nextValue = rowWindow.getRow(i + 1).getInt(0); - x1 = currentTime - lastTime; - x2 = nextTime - currentTime; - y1 = currentValue - lastValue; - y2 = nextValue - currentValue; + x1 = timeDifferenceAsDouble(currentTime, lastTime); + x2 = timeDifferenceAsDouble(nextTime, currentTime); + y1 = (double) currentValue - lastValue; + y2 = (double) nextValue - currentValue; - value = - (x1 * x2 + y1 * y2) - / (Math.sqrt((double) x1 * x1 + y1 * y1) * Math.sqrt((double) x2 * x2 + y2 * y2)); + value = (x1 * x2 + y1 * y2) / (Math.sqrt(x1 * x1 + y1 * y1) * Math.sqrt(x2 * x2 + y2 * y2)); addToMaxHeap(pq, i, value); } @@ -341,8 +354,9 @@ public void outlierSampleLong(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> -o.right)); - long lastTime, currentTime, nextTime, x1, x2; - long lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + long lastValue, currentValue, nextValue; + double x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -354,14 +368,12 @@ public void outlierSampleLong(RowWindow rowWindow, PointCollector collector) currentValue = rowWindow.getRow(i).getLong(0); nextValue = rowWindow.getRow(i + 1).getLong(0); - x1 = currentTime - lastTime; - x2 = nextTime - currentTime; - y1 = currentValue - lastValue; - y2 = nextValue - currentValue; + x1 = timeDifferenceAsDouble(currentTime, lastTime); + x2 = timeDifferenceAsDouble(nextTime, currentTime); + y1 = timeDifferenceAsDouble(currentValue, lastValue); + y2 = timeDifferenceAsDouble(nextValue, currentValue); - value = - (x1 * x2 + y1 * y2) - / (Math.sqrt((double) x1 * x1 + y1 * y1) * Math.sqrt((double) x2 * x2 + y2 * y2)); + value = (x1 * x2 + y1 * y2) / (Math.sqrt(x1 * x1 + y1 * y1) * Math.sqrt(x2 * x2 + y2 * y2)); addToMaxHeap(pq, i, value); } @@ -381,8 +393,9 @@ public void outlierSampleFloat(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> -o.right)); - long lastTime, currentTime, nextTime, x1, x2; - float lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + float lastValue, currentValue, nextValue; + double x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -394,10 +407,10 @@ public void outlierSampleFloat(RowWindow rowWindow, PointCollector collector) currentValue = rowWindow.getRow(i).getFloat(0); nextValue = rowWindow.getRow(i + 1).getFloat(0); - x1 = currentTime - lastTime; - x2 = nextTime - currentTime; - y1 = currentValue - lastValue; - y2 = nextValue - currentValue; + x1 = timeDifferenceAsDouble(currentTime, lastTime); + x2 = timeDifferenceAsDouble(nextTime, currentTime); + y1 = (double) currentValue - lastValue; + y2 = (double) nextValue - currentValue; value = (x1 * x2 + y1 * y2) / (Math.sqrt(x1 * x1 + y1 * y1) * Math.sqrt(x2 * x2 + y2 * y2)); @@ -419,8 +432,8 @@ public void outlierSampleDouble(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> -o.right)); - long lastTime, currentTime, nextTime, x1, x2; - double lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + double lastValue, currentValue, nextValue, x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -432,8 +445,8 @@ public void outlierSampleDouble(RowWindow rowWindow, PointCollector collector) currentValue = rowWindow.getRow(i).getDouble(0); nextValue = rowWindow.getRow(i + 1).getDouble(0); - x1 = currentTime - lastTime; - x2 = nextTime - currentTime; + x1 = timeDifferenceAsDouble(currentTime, lastTime); + x2 = timeDifferenceAsDouble(nextTime, currentTime); y1 = currentValue - lastValue; y2 = nextValue - currentValue; @@ -458,8 +471,9 @@ public void outlierSampleInt(RowWindow rowWindow, PointCollector collector) thro PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> o.right)); - long lastTime, currentTime, nextTime, x1, x2; - int lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + int lastValue, currentValue, nextValue; + double x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -471,10 +485,10 @@ public void outlierSampleInt(RowWindow rowWindow, PointCollector collector) thro currentValue = rowWindow.getRow(i).getInt(0); nextValue = rowWindow.getRow(i + 1).getInt(0); - x1 = Math.abs(currentTime - lastTime); - x2 = Math.abs(nextTime - currentTime); - y1 = Math.abs(currentValue - lastValue); - y2 = Math.abs(nextValue - currentValue); + x1 = timeDistanceAsDouble(currentTime, lastTime); + x2 = timeDistanceAsDouble(nextTime, currentTime); + y1 = Math.abs((double) currentValue - lastValue); + y2 = Math.abs((double) nextValue - currentValue); value = (double) x1 + y1 + x2 + y2; @@ -495,8 +509,9 @@ public void outlierSampleLong(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> o.right)); - long lastTime, currentTime, nextTime, x1, x2; - long lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + long lastValue, currentValue, nextValue; + double x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -508,10 +523,10 @@ public void outlierSampleLong(RowWindow rowWindow, PointCollector collector) currentValue = rowWindow.getRow(i).getLong(0); nextValue = rowWindow.getRow(i + 1).getLong(0); - x1 = Math.abs(currentTime - lastTime); - x2 = Math.abs(nextTime - currentTime); - y1 = Math.abs(currentValue - lastValue); - y2 = Math.abs(nextValue - currentValue); + x1 = timeDistanceAsDouble(currentTime, lastTime); + x2 = timeDistanceAsDouble(nextTime, currentTime); + y1 = Math.abs(timeDifferenceAsDouble(currentValue, lastValue)); + y2 = Math.abs(timeDifferenceAsDouble(nextValue, currentValue)); value = (double) x1 + y1 + x2 + y2; @@ -532,8 +547,9 @@ public void outlierSampleFloat(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> o.right)); - long lastTime, currentTime, nextTime, x1, x2; - float lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + float lastValue, currentValue, nextValue; + double x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -545,10 +561,10 @@ public void outlierSampleFloat(RowWindow rowWindow, PointCollector collector) currentValue = rowWindow.getRow(i).getFloat(0); nextValue = rowWindow.getRow(i + 1).getFloat(0); - x1 = Math.abs(currentTime - lastTime); - x2 = Math.abs(nextTime - currentTime); - y1 = Math.abs(currentValue - lastValue); - y2 = Math.abs(nextValue - currentValue); + x1 = timeDistanceAsDouble(currentTime, lastTime); + x2 = timeDistanceAsDouble(nextTime, currentTime); + y1 = Math.abs((double) currentValue - lastValue); + y2 = Math.abs((double) nextValue - currentValue); value = x1 + y1 + x2 + y2; @@ -569,7 +585,8 @@ public void outlierSampleDouble(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> o.right)); - long lastTime, currentTime, nextTime, x1, x2; + long lastTime, currentTime, nextTime; + double x1, x2; double lastValue, currentValue, nextValue, y1, y2; double value; @@ -582,8 +599,8 @@ public void outlierSampleDouble(RowWindow rowWindow, PointCollector collector) currentValue = rowWindow.getRow(i).getDouble(0); nextValue = rowWindow.getRow(i + 1).getDouble(0); - x1 = Math.abs(currentTime - lastTime); - x2 = Math.abs(nextTime - currentTime); + x1 = timeDistanceAsDouble(currentTime, lastTime); + x2 = timeDistanceAsDouble(nextTime, currentTime); y1 = Math.abs(currentValue - lastValue); y2 = Math.abs(nextValue - currentValue); @@ -844,4 +861,8 @@ public boolean isWindowSizeTooSmallDouble( } return false; } + + private static double timeDistanceAsDouble(long left, long right) { + return Math.abs(timeDifferenceAsDouble(left, right)); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTimeDifference.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTimeDifference.java index 6478eaf7fd74..578eca3bc54a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTimeDifference.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTimeDifference.java @@ -29,6 +29,8 @@ import java.io.IOException; +import static com.google.common.math.LongMath.saturatedSubtract; + public class UDTFTimeDifference implements UDTF { private boolean hasPrevious = false; @@ -49,7 +51,7 @@ public void transform(Row row, PointCollector collector) throws IOException { } long currentTime = row.getTime(); - collector.putLong(currentTime, currentTime - previousTime); + collector.putLong(currentTime, saturatedSubtract(currentTime, previousTime)); previousTime = currentTime; } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/CommonDateTimeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/CommonDateTimeUtils.java index b4f03f111505..9ea1c87b75e2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/CommonDateTimeUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/CommonDateTimeUtils.java @@ -24,6 +24,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.math.BigInteger; import java.time.Duration; import java.util.function.BiConsumer; @@ -68,6 +69,25 @@ public static long currentTime() { } } + public static double timeDifferenceAsDouble(long left, long right) { + try { + return Math.subtractExact(left, right); + } catch (ArithmeticException e) { + return BigInteger.valueOf(left).subtract(BigInteger.valueOf(right)).doubleValue(); + } + } + + /** Converts a potentially wider integer result to a long without wrapping at either bound. */ + public static long saturateToLong(BigInteger value) { + if (value.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + return Long.MAX_VALUE; + } + if (value.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) { + return Long.MIN_VALUE; + } + return value.longValue(); + } + public static String convertMillisecondToDurationStr(long millisecond) { StringBuilder stringBuilder = new StringBuilder(); boolean minus = false; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java index 0dc6eed8af40..00d3e46e1847 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java @@ -25,50 +25,59 @@ import java.math.BigInteger; +import static com.google.common.math.LongMath.saturatedAdd; +import static org.apache.iotdb.commons.utils.CommonDateTimeUtils.saturateToLong; + public class TimePartitionUtils { /** * Time partition origin for dividing database, the time unit is the same with IoTDB's * TimestampPrecision */ - private static long timePartitionOrigin = + private static volatile long timePartitionOrigin = CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin(); /** Time range for dividing database, the time unit is the same with IoTDB's TimestampPrecision */ - private static long timePartitionInterval = + private static volatile long timePartitionInterval = CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); - private static final BigInteger bigTimePartitionOrigin = BigInteger.valueOf(timePartitionOrigin); - private static final BigInteger bigTimePartitionInterval = - BigInteger.valueOf(timePartitionInterval); - private static final boolean originMayCauseOverflow = (timePartitionOrigin != 0); - private static final long timePartitionLowerBoundWithoutOverflow; - private static final long timePartitionUpperBoundWithoutOverflow; + private static final BigInteger BIG_LONG_MIN = BigInteger.valueOf(Long.MIN_VALUE); + private static final BigInteger BIG_LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); + private static final BigInteger BIG_ONE = BigInteger.ONE; + + private static volatile long timePartitionLowerBoundWithoutOverflow; + private static volatile long timePartitionUpperBoundWithoutOverflow; + private static volatile boolean timePartitionLowerBoundOverflow; + private static volatile boolean timePartitionUpperBoundOverflow; static { - long minPartition = getTimePartitionIdWithoutOverflow(Long.MIN_VALUE); - long maxPartition = getTimePartitionIdWithoutOverflow(Long.MAX_VALUE); - BigInteger minPartitionStartTime = - BigInteger.valueOf(minPartition) - .multiply(bigTimePartitionInterval) - .add(bigTimePartitionOrigin); - BigInteger maxPartitionEndTime = - BigInteger.valueOf(maxPartition) - .multiply(bigTimePartitionInterval) - .add(bigTimePartitionInterval) - .add(bigTimePartitionOrigin); - if (minPartitionStartTime.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) { - timePartitionLowerBoundWithoutOverflow = - minPartitionStartTime.add(bigTimePartitionInterval).longValue(); - } else { - timePartitionLowerBoundWithoutOverflow = minPartitionStartTime.longValue(); + updateTimePartitionBound(); + } + + private static void updateTimePartitionBound() { + BigInteger minPartition = getTimePartitionIdAsBigInteger(Long.MIN_VALUE); + BigInteger maxPartition = getTimePartitionIdAsBigInteger(Long.MAX_VALUE); + timePartitionLowerBoundOverflow = minPartition.compareTo(BIG_LONG_MIN) < 0; + timePartitionUpperBoundOverflow = maxPartition.compareTo(BIG_LONG_MAX) > 0; + + BigInteger firstRepresentablePartition = minPartition.max(BIG_LONG_MIN); + BigInteger firstRepresentableStart = + getTimePartitionStartTimeAsBigInteger(firstRepresentablePartition); + if (timePartitionLowerBoundOverflow || firstRepresentableStart.compareTo(BIG_LONG_MIN) < 0) { + firstRepresentableStart = + firstRepresentableStart.add(BigInteger.valueOf(timePartitionInterval)); } - if (maxPartitionEndTime.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { - timePartitionUpperBoundWithoutOverflow = - maxPartitionEndTime.subtract(bigTimePartitionInterval).longValue(); + timePartitionLowerBoundWithoutOverflow = saturateToLong(firstRepresentableStart); + + BigInteger lastRepresentablePartition = maxPartition.min(BIG_LONG_MAX); + BigInteger lastRepresentableStart; + if (timePartitionUpperBoundOverflow) { + lastRepresentableStart = + getTimePartitionStartTimeAsBigInteger(lastRepresentablePartition.add(BIG_ONE)); } else { - timePartitionUpperBoundWithoutOverflow = maxPartitionEndTime.longValue(); + lastRepresentableStart = getTimePartitionStartTimeAsBigInteger(lastRepresentablePartition); } + timePartitionUpperBoundWithoutOverflow = saturateToLong(lastRepresentableStart); } public static TTimePartitionSlot getTimePartitionSlot(long time) { @@ -85,94 +94,131 @@ public static long getTimePartitionLowerBound(long time) { if (time < timePartitionLowerBoundWithoutOverflow) { return Long.MIN_VALUE; } - if (originMayCauseOverflow) { - return BigInteger.valueOf(getTimePartitionIdWithoutOverflow(time)) - .multiply(bigTimePartitionInterval) - .add(bigTimePartitionOrigin) - .longValue(); - } else { - return getTimePartitionId(time) * timePartitionInterval + timePartitionOrigin; + if (time >= timePartitionUpperBoundWithoutOverflow) { + return timePartitionUpperBoundWithoutOverflow; } + return getTimePartitionStartTime(getTimePartitionId(time)); } public static long getTimePartitionUpperBound(long time) { if (time >= timePartitionUpperBoundWithoutOverflow) { return Long.MAX_VALUE; } - long lowerBound = getTimePartitionLowerBound(time); - return lowerBound == Long.MIN_VALUE - ? timePartitionLowerBoundWithoutOverflow - : lowerBound + timePartitionInterval; + if (time < timePartitionLowerBoundWithoutOverflow) { + return timePartitionLowerBoundWithoutOverflow; + } + return saturatedAdd(getTimePartitionLowerBound(time), timePartitionInterval); + } + + public static long getTimePartitionEndTime(long time) { + long upperBound = getTimePartitionUpperBound(time); + if (upperBound != Long.MAX_VALUE) { + return upperBound - 1; + } + return getTimePartitionLowerBound(time) == getTimePartitionLowerBound(Long.MAX_VALUE) + ? Long.MAX_VALUE + : Long.MAX_VALUE - 1; + } + + public static boolean isAfterOrEqualToTimePartitionUpperBound( + long time, long timePartitionStartTime, long timePartitionUpperBound) { + if (timePartitionUpperBound != Long.MAX_VALUE) { + return time >= timePartitionUpperBound; + } + return time == Long.MAX_VALUE && getTimePartitionLowerBound(time) != timePartitionStartTime; + } + + public static boolean isTimePartitionStartTime(long time) { + return getTimePartitionLowerBound(time) == time; } public static long getTimePartitionId(long time) { - time -= timePartitionOrigin; - return time > 0 || time % timePartitionInterval == 0 - ? time / timePartitionInterval - : time / timePartitionInterval - 1; + final long timeFromOrigin; + try { + timeFromOrigin = Math.subtractExact(time, timePartitionOrigin); + } catch (ArithmeticException e) { + return getTimePartitionIdWithoutOverflow(time); + } + return Math.floorDiv(timeFromOrigin, timePartitionInterval); } public static long getTimePartitionIdWithoutOverflow(long time) { - BigInteger bigTime = BigInteger.valueOf(time).subtract(bigTimePartitionOrigin); - BigInteger partitionId = - bigTime.compareTo(BigInteger.ZERO) > 0 - || bigTime.remainder(bigTimePartitionInterval).equals(BigInteger.ZERO) - ? bigTime.divide(bigTimePartitionInterval) - : bigTime.divide(bigTimePartitionInterval).subtract(BigInteger.ONE); + BigInteger partitionId = getTimePartitionIdAsBigInteger(time); + if (partitionId.compareTo(BIG_LONG_MIN) < 0) { + return Long.MIN_VALUE; + } + if (partitionId.compareTo(BIG_LONG_MAX) > 0) { + return Long.MAX_VALUE; + } return partitionId.longValue(); } + private static BigInteger getTimePartitionIdAsBigInteger(long time) { + BigInteger bigTime = BigInteger.valueOf(time).subtract(BigInteger.valueOf(timePartitionOrigin)); + BigInteger bigTimePartitionInterval = BigInteger.valueOf(timePartitionInterval); + return bigTime.compareTo(BigInteger.ZERO) > 0 + || bigTime.remainder(bigTimePartitionInterval).equals(BigInteger.ZERO) + ? bigTime.divide(bigTimePartitionInterval) + : bigTime.divide(bigTimePartitionInterval).subtract(BigInteger.ONE); + } + public static long getStartTimeByPartitionId(long partitionId) { - return (partitionId * timePartitionInterval) + timePartitionOrigin; + return getTimePartitionStartTime(partitionId); } public static boolean satisfyPartitionId(long startTime, long endTime, long partitionId) { - long startPartition = - originMayCauseOverflow - ? getTimePartitionIdWithoutOverflow(startTime) - : getTimePartitionId(startTime); - long endPartition = - originMayCauseOverflow - ? getTimePartitionIdWithoutOverflow(endTime) - : getTimePartitionId(endTime); + long startPartition = getTimePartitionId(startTime); + long endPartition = getTimePartitionId(endTime); return startPartition <= partitionId && endPartition >= partitionId; } public static boolean satisfyPartitionStartTime(Filter timeFilter, long partitionStartTime) { - long partitionEndTime = - partitionStartTime >= timePartitionLowerBoundWithoutOverflow - ? Long.MAX_VALUE - : (partitionStartTime + timePartitionInterval - 1); - return timeFilter == null - || timeFilter.satisfyStartEndTime(partitionStartTime, partitionEndTime); + if (timeFilter == null) { + return true; + } + long partitionEndTime = getTimePartitionEndTime(partitionStartTime); + return timeFilter.satisfyStartEndTime(partitionStartTime, partitionEndTime); } public static boolean satisfyTimePartition(Filter timeFilter, long partitionId) { - long partitionStartTime; - if (originMayCauseOverflow) { - partitionStartTime = - BigInteger.valueOf(partitionId) - .multiply(bigTimePartitionInterval) - .add(bigTimePartitionOrigin) - .longValue(); - } else { - partitionStartTime = partitionId * timePartitionInterval + timePartitionOrigin; + return satisfyPartitionStartTime(timeFilter, getTimePartitionStartTime(partitionId)); + } + + private static long getTimePartitionStartTime(long partitionId) { + if (partitionId == Long.MIN_VALUE && timePartitionLowerBoundOverflow) { + return Long.MIN_VALUE; } - return satisfyPartitionStartTime(timeFilter, partitionStartTime); + if (partitionId == Long.MAX_VALUE && timePartitionUpperBoundOverflow) { + return timePartitionUpperBoundWithoutOverflow; + } + return saturateToLong(getTimePartitionStartTimeAsBigInteger(BigInteger.valueOf(partitionId))); + } + + private static BigInteger getTimePartitionStartTimeAsBigInteger(BigInteger partitionId) { + return partitionId + .multiply(BigInteger.valueOf(timePartitionInterval)) + .add(BigInteger.valueOf(timePartitionOrigin)); } public static void setTimePartitionInterval(long timePartitionInterval) { TimePartitionUtils.timePartitionInterval = timePartitionInterval; + updateTimePartitionBound(); + } + + public static void setTimePartitionOrigin(long timePartitionOrigin) { + TimePartitionUtils.timePartitionOrigin = timePartitionOrigin; + updateTimePartitionBound(); } public static long getEstimateTimePartitionSize(long startTime, long endTime) { - if (endTime > 0 && startTime < 0) { - return BigInteger.valueOf(endTime) - .subtract(BigInteger.valueOf(startTime)) - .divide(bigTimePartitionInterval) - .longValue() - + 1; + BigInteger estimateSize = + BigInteger.valueOf(endTime) + .subtract(BigInteger.valueOf(startTime)) + .divide(BigInteger.valueOf(timePartitionInterval)) + .add(BigInteger.ONE); + if (estimateSize.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + return Long.MAX_VALUE; } - return (endTime - startTime) / timePartitionInterval + 1; + return estimateSize.longValue(); } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/partition/SeriesPartitionTableTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/partition/SeriesPartitionTableTest.java index ab63deb3c68f..85cb63bd38c4 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/partition/SeriesPartitionTableTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/partition/SeriesPartitionTableTest.java @@ -108,4 +108,47 @@ public void snapshotSerDeTest() throws TException, IOException { table1.deserialize(inputStream, protocol); Assert.assertEquals(table0, table1); } + + @Test + public void autoCleanPartitionTableShouldNotExpireOnOverflow() { + TConsensusGroupId consensusGroupId = new TConsensusGroupId(TConsensusGroupType.DataRegion, 0); + + SeriesPartitionTable table = new SeriesPartitionTable(); + TTimePartitionSlot nearMaxSlot = new TTimePartitionSlot(Long.MAX_VALUE - 1); + table.putDataPartition(nearMaxSlot, consensusGroupId); + Assert.assertTrue( + table.autoCleanPartitionTable(0, new TTimePartitionSlot(Long.MAX_VALUE - 1)).isEmpty()); + Assert.assertTrue(table.getSeriesPartitionMap().containsKey(nearMaxSlot)); + + table = new SeriesPartitionTable(); + TTimePartitionSlot normalSlot = new TTimePartitionSlot(0); + table.putDataPartition(normalSlot, consensusGroupId); + Assert.assertTrue( + table + .autoCleanPartitionTable(Long.MAX_VALUE - 1, new TTimePartitionSlot(Long.MAX_VALUE - 1)) + .isEmpty()); + Assert.assertTrue(table.getSeriesPartitionMap().containsKey(normalSlot)); + } + + @Test + public void getTimeSlotListShouldIncludeLongMaxSlotWithUnboundedEndTime() { + final TConsensusGroupId consensusGroupId = + new TConsensusGroupId(TConsensusGroupType.DataRegion, 0); + final TConsensusGroupId allRegionId = new TConsensusGroupId(TConsensusGroupType.DataRegion, -1); + final TTimePartitionSlot previousSlot = new TTimePartitionSlot(Long.MAX_VALUE - 1); + final TTimePartitionSlot lastSlot = new TTimePartitionSlot(Long.MAX_VALUE); + + final SeriesPartitionTable table = new SeriesPartitionTable(); + table.putDataPartition(previousSlot, consensusGroupId); + table.putDataPartition(lastSlot, consensusGroupId); + + final List expected = new ArrayList<>(); + expected.add(previousSlot); + expected.add(lastSlot); + + Assert.assertEquals( + expected, table.getTimeSlotList(allRegionId, Long.MAX_VALUE - 1, Long.MAX_VALUE)); + Assert.assertEquals( + expected, table.getTimeSlotList(consensusGroupId, Long.MAX_VALUE - 1, Long.MAX_VALUE)); + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/CommonDateTimeUtilsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/CommonDateTimeUtilsTest.java index 9ae8197ea640..2a3430047730 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/CommonDateTimeUtilsTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/CommonDateTimeUtilsTest.java @@ -22,7 +22,11 @@ import org.junit.Assert; import org.junit.Test; +import java.math.BigInteger; + import static org.apache.iotdb.commons.utils.CommonDateTimeUtils.convertMillisecondToDurationStr; +import static org.apache.iotdb.commons.utils.CommonDateTimeUtils.saturateToLong; +import static org.apache.iotdb.commons.utils.CommonDateTimeUtils.timeDifferenceAsDouble; public class CommonDateTimeUtilsTest { @Test @@ -38,4 +42,33 @@ public void convertMillisecondToDurationStrTest() { convertMillisecondToDurationStr(99999999999999999L)); Assert.assertEquals("-(10 second 86 ms)", convertMillisecondToDurationStr(-10086)); } + + @Test + public void timeDifferenceAsDoublePreservesSmallDeltasAtLargeTimestamps() { + long timestamp = 1L << 60; + + Assert.assertEquals(50.0, timeDifferenceAsDouble(timestamp + 50, timestamp), 0.0); + Assert.assertEquals(100.0, timeDifferenceAsDouble(timestamp + 100, timestamp), 0.0); + Assert.assertEquals( + 0.5, + timeDifferenceAsDouble(timestamp + 50, timestamp) + / timeDifferenceAsDouble(timestamp + 100, timestamp), + 0.0); + } + + @Test + public void timeDifferenceAsDoubleHandlesLongOverflow() { + Assert.assertEquals( + Math.scalb(1.0, 64) - 1, + timeDifferenceAsDouble(Long.MAX_VALUE, Long.MIN_VALUE), + Math.scalb(1.0, 11)); + } + + @Test + public void saturateToLongClampsAtBothBounds() { + Assert.assertEquals(Long.MAX_VALUE, saturateToLong(BigInteger.TWO.pow(63))); + Assert.assertEquals( + Long.MIN_VALUE, saturateToLong(BigInteger.TWO.pow(63).negate().subtract(BigInteger.ONE))); + Assert.assertEquals(123L, saturateToLong(BigInteger.valueOf(123L))); + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java index ea0eeda45d28..9f1b3f327fbe 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java @@ -22,6 +22,8 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.tsfile.read.filter.factory.TimeFilterApi; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -33,12 +35,31 @@ public class TimePartitionUtilsTest { private static final long TEST_TIME_PARTITION_ORIGIN = 1000L; private static final long TEST_TIME_PARTITION_INTERVAL = 3600000L; + private long previousTimePartitionOrigin; + private long previousTimePartitionInterval; + @Before public void setUp() { + previousTimePartitionOrigin = + CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin(); + previousTimePartitionInterval = + CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(TEST_TIME_PARTITION_ORIGIN); CommonDescriptor.getInstance() .getConfig() .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL); + TimePartitionUtils.setTimePartitionOrigin(TEST_TIME_PARTITION_ORIGIN); + TimePartitionUtils.setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL); + } + + @After + public void tearDown() { + CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(previousTimePartitionOrigin); + CommonDescriptor.getInstance() + .getConfig() + .setTimePartitionInterval(previousTimePartitionInterval); + TimePartitionUtils.setTimePartitionOrigin(previousTimePartitionOrigin); + TimePartitionUtils.setTimePartitionInterval(previousTimePartitionInterval); } @Test @@ -105,4 +126,143 @@ public void testOverflow() { long upperBound = TimePartitionUtils.getTimePartitionUpperBound(testTime); assertEquals(Long.MAX_VALUE, upperBound); } + + @Test + public void testIsTimePartitionStartTimeWithOrigin() { + Assert.assertTrue(TimePartitionUtils.isTimePartitionStartTime(TEST_TIME_PARTITION_ORIGIN)); + Assert.assertFalse(TimePartitionUtils.isTimePartitionStartTime(TEST_TIME_PARTITION_ORIGIN + 1)); + Assert.assertTrue( + TimePartitionUtils.isTimePartitionStartTime( + TEST_TIME_PARTITION_ORIGIN + TEST_TIME_PARTITION_INTERVAL)); + } + + @Test + public void testSatisfyPartitionStartTimeWithNormalPartitionEnd() { + Assert.assertFalse( + TimePartitionUtils.satisfyPartitionStartTime( + TimeFilterApi.gtEq(TEST_TIME_PARTITION_ORIGIN + TEST_TIME_PARTITION_INTERVAL), + TEST_TIME_PARTITION_ORIGIN)); + Assert.assertFalse( + TimePartitionUtils.satisfyTimePartition( + TimeFilterApi.gtEq(TEST_TIME_PARTITION_ORIGIN + TEST_TIME_PARTITION_INTERVAL), 0)); + Assert.assertTrue( + TimePartitionUtils.satisfyPartitionStartTime( + TimeFilterApi.gtEq(TEST_TIME_PARTITION_ORIGIN + TEST_TIME_PARTITION_INTERVAL - 1), + TEST_TIME_PARTITION_ORIGIN)); + } + + @Test + public void testSatisfyPartitionStartTimeWithOverflowPartitionEnd() { + long partitionStartTime = TimePartitionUtils.getTimePartitionSlot(Long.MAX_VALUE).startTime; + + Assert.assertTrue( + TimePartitionUtils.satisfyPartitionStartTime( + TimeFilterApi.eq(Long.MAX_VALUE), partitionStartTime)); + } + + @Test + public void testPartitionEndTimeAndUpperBoundCheckWithOverflowPartitionEnd() { + long partitionStartTime = TimePartitionUtils.getTimePartitionSlot(Long.MAX_VALUE).startTime; + long upperBound = TimePartitionUtils.getTimePartitionUpperBound(partitionStartTime); + + assertEquals(Long.MAX_VALUE, TimePartitionUtils.getTimePartitionEndTime(partitionStartTime)); + assertEquals(Long.MAX_VALUE, upperBound); + Assert.assertFalse( + TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + Long.MAX_VALUE, partitionStartTime, upperBound)); + Assert.assertTrue( + TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + TEST_TIME_PARTITION_ORIGIN + TEST_TIME_PARTITION_INTERVAL, + TEST_TIME_PARTITION_ORIGIN, + TimePartitionUtils.getTimePartitionUpperBound(TEST_TIME_PARTITION_ORIGIN))); + } + + @Test + public void testExactLongMaxUpperBoundCheck() { + CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(0); + CommonDescriptor.getInstance().getConfig().setTimePartitionInterval(1); + TimePartitionUtils.setTimePartitionOrigin(0); + TimePartitionUtils.setTimePartitionInterval(1); + + assertEquals(Long.MAX_VALUE, TimePartitionUtils.getTimePartitionUpperBound(Long.MAX_VALUE - 1)); + assertEquals( + Long.MAX_VALUE - 1, TimePartitionUtils.getTimePartitionEndTime(Long.MAX_VALUE - 1)); + assertEquals(Long.MAX_VALUE, TimePartitionUtils.getTimePartitionEndTime(Long.MAX_VALUE)); + Assert.assertTrue( + TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + Long.MAX_VALUE, Long.MAX_VALUE - 1, Long.MAX_VALUE)); + Assert.assertFalse( + TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE)); + } + + @Test + public void testSatisfyTimePartitionWithOverflowPartitionStart() { + long partitionId = TimePartitionUtils.getTimePartitionIdWithoutOverflow(Long.MIN_VALUE); + long nextPartitionStartTime = TimePartitionUtils.getTimePartitionUpperBound(Long.MIN_VALUE); + + Assert.assertTrue( + TimePartitionUtils.satisfyTimePartition(TimeFilterApi.eq(Long.MIN_VALUE), partitionId)); + Assert.assertFalse( + TimePartitionUtils.satisfyTimePartition( + TimeFilterApi.eq(nextPartitionStartTime), partitionId)); + } + + @Test + public void testExactLongMinPartitionStartUpperBound() { + CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(0); + CommonDescriptor.getInstance().getConfig().setTimePartitionInterval(1); + TimePartitionUtils.setTimePartitionOrigin(0); + TimePartitionUtils.setTimePartitionInterval(1); + + assertEquals(Long.MIN_VALUE, TimePartitionUtils.getTimePartitionLowerBound(Long.MIN_VALUE)); + assertEquals(Long.MIN_VALUE + 1, TimePartitionUtils.getTimePartitionUpperBound(Long.MIN_VALUE)); + assertEquals(Long.MIN_VALUE, TimePartitionUtils.getTimePartitionEndTime(Long.MIN_VALUE)); + Assert.assertTrue( + TimePartitionUtils.satisfyPartitionStartTime( + TimeFilterApi.eq(Long.MIN_VALUE), Long.MIN_VALUE)); + Assert.assertFalse( + TimePartitionUtils.satisfyPartitionStartTime( + TimeFilterApi.eq(Long.MIN_VALUE + 1), Long.MIN_VALUE)); + } + + @Test + public void testGetTimePartitionIdWithOverflowOrigin() { + assertEquals( + TimePartitionUtils.getTimePartitionIdWithoutOverflow(Long.MIN_VALUE), + TimePartitionUtils.getTimePartitionId(Long.MIN_VALUE)); + assertEquals( + TimePartitionUtils.getTimePartitionIdWithoutOverflow(Long.MAX_VALUE), + TimePartitionUtils.getTimePartitionId(Long.MAX_VALUE)); + } + + @Test + public void testLongMinPartitionWithUnitIntervalAndNonZeroOrigin() { + CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(1); + CommonDescriptor.getInstance().getConfig().setTimePartitionInterval(1); + TimePartitionUtils.setTimePartitionOrigin(1); + TimePartitionUtils.setTimePartitionInterval(1); + + assertEquals(Long.MIN_VALUE, TimePartitionUtils.getTimePartitionId(Long.MIN_VALUE)); + assertEquals(Long.MIN_VALUE, TimePartitionUtils.getTimePartitionSlot(Long.MIN_VALUE).startTime); + assertEquals( + Long.MIN_VALUE, TimePartitionUtils.getTimePartitionSlot(Long.MIN_VALUE + 1).startTime); + assertEquals(Long.MIN_VALUE + 2, TimePartitionUtils.getTimePartitionUpperBound(Long.MIN_VALUE)); + assertEquals(Long.MIN_VALUE + 1, TimePartitionUtils.getTimePartitionEndTime(Long.MIN_VALUE)); + assertEquals( + Long.MIN_VALUE + 2, TimePartitionUtils.getTimePartitionSlot(Long.MIN_VALUE + 2).startTime); + } + + @Test + public void testGetEstimateTimePartitionSizeWithOverflow() { + long previousTimePartitionInterval = TimePartitionUtils.getTimePartitionInterval(); + try { + TimePartitionUtils.setTimePartitionInterval(1); + assertEquals( + Long.MAX_VALUE, + TimePartitionUtils.getEstimateTimePartitionSize(Long.MIN_VALUE, Long.MAX_VALUE)); + } finally { + TimePartitionUtils.setTimePartitionInterval(previousTimePartitionInterval); + } + } }