diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index 90cb706c0d5f..5040980e0c7a 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -134,18 +134,13 @@ public Sequence runSegmentMetadataQuery(Iterable seg return Sequences.simple( Lists.transform( Lists.newArrayList(segments), - (segment) -> new SegmentAnalysis( - segment.toString(), - ImmutableList.of(segment.getInterval()), - columnToAnalysisMap, - 40, - 40, - null, - null, - null, - null, - false - ) + (segment) -> new SegmentAnalysis.Builder(segment) + .interval(segment.getInterval()) + .columns(columnToAnalysisMap) + .size(40) + .numRows(40) + .rollup(false) + .build() ) ); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java index 8a8d403e1605..cd78e2bef26b 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java @@ -126,6 +126,7 @@ import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.data.ListIndexed; import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; +import org.apache.druid.segment.file.NoopSegmentFileMapper; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; import org.apache.druid.segment.indexing.BatchIOConfig; import org.apache.druid.segment.indexing.CombinedDataSchema; @@ -2354,7 +2355,7 @@ private static class TestIndexIO extends IndexIO new ListIndexed<>(segment.getDimensions()), null, columnMap, - null + NoopSegmentFileMapper.INSTANCE ) { @Override diff --git a/processing/src/main/java/org/apache/druid/java/util/common/io/smoosh/SmooshedFileMapper.java b/processing/src/main/java/org/apache/druid/java/util/common/io/smoosh/SmooshedFileMapper.java index 97af92438add..fdcefc382d71 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/io/smoosh/SmooshedFileMapper.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/io/smoosh/SmooshedFileMapper.java @@ -27,7 +27,9 @@ import org.apache.druid.java.util.common.ByteBufferUtils; import org.apache.druid.java.util.common.ISE; import org.apache.druid.segment.file.SegmentFileMapper; +import org.apache.druid.segment.file.SegmentFileMetadata; +import javax.annotation.Nullable; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -120,6 +122,14 @@ public Set getInternalFilenames() return internalFiles.keySet(); } + @Nullable + @Override + public SegmentFileMetadata getSegmentFileMetadata() + { + // legacy smoosh files have no container/bundle structure to report. + return null; + } + @Override public ByteBuffer mapFile(String name) throws IOException { diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java index 84f3cfd534df..8d5577811a07 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java @@ -55,6 +55,7 @@ import org.apache.druid.query.metadata.metadata.AggregatorMergeStrategy; import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentAnalysis.ContainerAnalysis; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.timeline.LogicalSegment; @@ -457,35 +458,68 @@ public static SegmentAnalysis mergeAnalyses( projections = null; } - return new SegmentAnalysis( - mergedId, - newIntervals, - columns, - arg1.getSize() + arg2.getSize(), - arg1.getNumRows() + arg2.getNumRows(), - aggregators.isEmpty() ? null : aggregators, - (projections == null || projections.isEmpty()) ? null : projections, - timestampSpec, - queryGranularity, - rollup - ); + // Merged containers report one total per bundle name rather than a raw list of every underlying segment's + // individual containers, since a container has no identity across segments (only bundle names do). + final List containers = mergeContainers(arg1.getContainers(), arg2.getContainers()); + + return new SegmentAnalysis.Builder(mergedId) + .intervals(newIntervals) + .columns(columns) + .size(arg1.getSize() + arg2.getSize()) + .numRows(arg1.getNumRows() + arg2.getNumRows()) + .aggregators(aggregators) + .projections(projections) + .timestampSpec(timestampSpec) + .queryGranularity(queryGranularity) + .rollup(rollup) + .containers(containers) + .build(); + } + + /** + * Sums container sizes by bundle name, since a container has no identity across segments. If only one side has + * data, it's returned unchanged instead of being run through the per-bundle collapse: there's nothing to merge it + * with, and a single segment's own list may already have more than one entry for the same bundle (a bundle + * spanning multiple containers). + */ + @Nullable + private static List mergeContainers( + @Nullable List containers1, + @Nullable List containers2 + ) + { + if (containers1 == null) { + return containers2; + } + if (containers2 == null) { + return containers1; + } + final Map sizeByBundle = new LinkedHashMap<>(); + for (ContainerAnalysis container : Iterables.concat(containers1, containers2)) { + sizeByBundle.merge(container.bundle(), container.size(), Long::sum); + } + final List merged = new ArrayList<>(sizeByBundle.size()); + for (Map.Entry entry : sizeByBundle.entrySet()) { + merged.add(new ContainerAnalysis(entry.getKey(), entry.getValue())); + } + return merged; } @VisibleForTesting public static SegmentAnalysis finalizeAnalysis(SegmentAnalysis analysis) { - return new SegmentAnalysis( - analysis.getId(), - analysis.getIntervals() != null ? JodaUtils.condenseIntervals(analysis.getIntervals()) : null, - analysis.getColumns(), - analysis.getSize(), - analysis.getNumRows(), - analysis.getAggregators(), - analysis.getProjections(), - analysis.getTimestampSpec(), - analysis.getQueryGranularity(), - analysis.isRollup() - ); + return new SegmentAnalysis.Builder(analysis.getId()) + .intervals(analysis.getIntervals() != null ? JodaUtils.condenseIntervals(analysis.getIntervals()) : null) + .columns(analysis.getColumns()) + .size(analysis.getSize()) + .numRows(analysis.getNumRows()) + .aggregators(analysis.getAggregators()) + .projections(analysis.getProjections()) + .timestampSpec(analysis.getTimestampSpec()) + .queryGranularity(analysis.getQueryGranularity()) + .rollup(analysis.isRollup()) + .containers(analysis.getContainers()) + .build(); } public SegmentMetadataQueryConfig getConfig() diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java index 6f88cff920dc..025515ad5974 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java @@ -36,10 +36,13 @@ import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.ColumnIncluderator; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentAnalysis.ContainerAnalysis; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.Metadata; +import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.Segment; +import org.apache.druid.segment.file.SegmentFileContainerMetadata; import org.joda.time.Interval; import javax.annotation.Nullable; @@ -165,22 +168,33 @@ public Sequence run(QueryPlus inQ, ResponseCon } } - return Sequences.simple( - Collections.singletonList( - new SegmentAnalysis( - segment.getId().toString(), - retIntervals, - columns, - totalSize, - numRows, - aggregators, - projectionsMap, - timestampSpec, - queryGranularity, - rollup - ) - ) - ); + final List containers; + if (updatedQuery.hasContainerSizes()) { + final QueryableIndex index = segment.as(QueryableIndex.class); + final List fileContainers = index == null ? null : index.getFileContainers(); + containers = fileContainers == null + ? null + : fileContainers.stream() + .map(c -> new ContainerAnalysis(c.getBundle(), c.getSize())) + .collect(Collectors.toList()); + } else { + containers = null; + } + + final SegmentAnalysis analysis = new SegmentAnalysis.Builder(segment.getId().toString()) + .intervals(retIntervals) + .columns(columns) + .size(totalSize) + .numRows(numRows) + .aggregators(aggregators) + .projections(projectionsMap) + .timestampSpec(timestampSpec) + .queryGranularity(queryGranularity) + .rollup(rollup) + .containers(containers) + .build(); + + return Sequences.simple(Collections.singletonList(analysis)); } }; } diff --git a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java index 8ba23be919e0..d4c7893ff554 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java @@ -28,6 +28,7 @@ import org.apache.druid.timeline.SegmentId; import org.joda.time.Interval; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -60,9 +61,31 @@ public class SegmentAnalysis implements Comparable private final TimestampSpec timestampSpec; private final Granularity queryGranularity; private final Boolean rollup; + private final List containers; - @JsonCreator + /** + * Retained for binary compatibility with code compiled against the pre-{@link ContainerAnalysis} constructor. + * New callers should use {@link Builder}. + */ + @Deprecated public SegmentAnalysis( + String id, + List interval, + LinkedHashMap columns, + long size, + long numRows, + Map aggregators, + Map projections, + TimestampSpec timestampSpec, + Granularity queryGranularity, + Boolean rollup + ) + { + this(id, interval, columns, size, numRows, aggregators, projections, timestampSpec, queryGranularity, rollup, null); + } + + @JsonCreator + SegmentAnalysis( @JsonProperty("id") String id, @JsonProperty("intervals") List interval, @JsonProperty("columns") LinkedHashMap columns, @@ -72,7 +95,8 @@ public SegmentAnalysis( @JsonProperty("projections") Map projections, @JsonProperty("timestampSpec") TimestampSpec timestampSpec, @JsonProperty("queryGranularity") Granularity queryGranularity, - @JsonProperty("rollup") Boolean rollup + @JsonProperty("rollup") Boolean rollup, + @JsonProperty("containers") List containers ) { this.id = id; @@ -85,6 +109,7 @@ public SegmentAnalysis( this.timestampSpec = timestampSpec; this.queryGranularity = queryGranularity; this.rollup = rollup; + this.containers = containers; } @JsonProperty @@ -147,6 +172,12 @@ public Map getProjections() return projections; } + @JsonProperty + public List getContainers() + { + return containers; + } + @Override public String toString() { @@ -161,6 +192,7 @@ public String toString() ", timestampSpec=" + timestampSpec + ", queryGranularity=" + queryGranularity + ", rollup=" + rollup + + ", containers=" + containers + '}'; } @@ -186,7 +218,8 @@ public boolean equals(Object o) Objects.equals(aggregators, that.aggregators) && Objects.equals(projections, that.projections) && Objects.equals(timestampSpec, that.timestampSpec) && - Objects.equals(queryGranularity, that.queryGranularity); + Objects.equals(queryGranularity, that.queryGranularity) && + Objects.equals(containers, that.containers); } /** @@ -206,7 +239,8 @@ public int hashCode() projections, timestampSpec, queryGranularity, - rollup + rollup, + containers ); } @@ -217,19 +251,26 @@ public int compareTo(SegmentAnalysis rhs) } /** - * Helper class to build {@link SegmentAnalysis} objects. + * Helper class to build {@link SegmentAnalysis} objects. Supports both incremental, single-entry building (handy + * for tests) and bulk setters that take an already-computed map/list (handy for production call sites that + * already have the whole thing on hand). */ public static class Builder { private final String segmentId; - private final LinkedHashMap columns = new LinkedHashMap<>(); - private final Map aggregators = new LinkedHashMap<>(); - private final Map projections = new LinkedHashMap<>(); + private LinkedHashMap columns = new LinkedHashMap<>(); + private Map aggregators = new LinkedHashMap<>(); + private Map projections = new LinkedHashMap<>(); + private List containers = new ArrayList<>(); private List intervals = null; - private Optional size = Optional.empty(); - private Optional numRows = Optional.empty(); + private Optional size = Optional.empty(); + private Optional numRows = Optional.empty(); private Optional rollup = Optional.empty(); + @Nullable + private TimestampSpec timestampSpec = null; + @Nullable + private Granularity queryGranularity = null; public Builder(String segmentId) { @@ -241,7 +282,7 @@ public Builder(SegmentId segmentId) this.segmentId = segmentId.toString(); } - public Builder size(int size) + public Builder size(long size) { if (this.size.isEmpty()) { this.size = Optional.of(size); @@ -251,7 +292,16 @@ public Builder size(int size) return this; } - public Builder numRows(int numRows) + /** + * Retained for binary compatibility with code compiled against the pre-{@link ContainerAnalysis} signature. + */ + @Deprecated + public Builder size(int size) + { + return size((long) size); + } + + public Builder numRows(long numRows) { if (this.numRows.isEmpty()) { this.numRows = Optional.of(numRows); @@ -261,8 +311,20 @@ public Builder numRows(int numRows) return this; } - public Builder rollup(boolean rollup) + /** + * Retained for binary compatibility with code compiled against the pre-{@link ContainerAnalysis} signature. + */ + @Deprecated + public Builder numRows(int numRows) + { + return numRows((long) numRows); + } + + public Builder rollup(@Nullable Boolean rollup) { + if (rollup == null) { + return this; + } if (this.rollup.isEmpty()) { this.rollup = Optional.of(rollup); } else { @@ -271,6 +333,15 @@ public Builder rollup(boolean rollup) return this; } + /** + * Retained for binary compatibility with code compiled against the pre-{@link ContainerAnalysis} signature. + */ + @Deprecated + public Builder rollup(boolean rollup) + { + return rollup(Boolean.valueOf(rollup)); + } + public Builder interval(Interval interval) { if (this.intervals == null) { @@ -280,38 +351,102 @@ public Builder interval(Interval interval) return this; } + public Builder intervals(@Nullable List intervals) + { + this.intervals = intervals == null ? null : new ArrayList<>(intervals); + return this; + } + public Builder column(String columnName, ColumnAnalysis columnAnalysis) { this.columns.put(columnName, columnAnalysis); return this; } + public Builder columns(@Nullable LinkedHashMap columns) + { + this.columns = columns == null ? new LinkedHashMap<>() : new LinkedHashMap<>(columns); + return this; + } + public Builder aggregator(String name, AggregatorFactory aggregatorFactory) { this.aggregators.put(name, aggregatorFactory); return this; } + public Builder aggregators(@Nullable Map aggregators) + { + this.aggregators = aggregators == null ? new LinkedHashMap<>() : new LinkedHashMap<>(aggregators); + return this; + } + public Builder projection(String name, AggregateProjectionMetadata projection) { this.projections.put(name, projection); return this; } + public Builder projections(@Nullable Map projections) + { + this.projections = projections == null ? new LinkedHashMap<>() : new LinkedHashMap<>(projections); + return this; + } + + public Builder container(String bundle, long size) + { + this.containers.add(new ContainerAnalysis(bundle, size)); + return this; + } + + public Builder containers(@Nullable List containers) + { + this.containers = containers == null ? new ArrayList<>() : new ArrayList<>(containers); + return this; + } + + public Builder timestampSpec(@Nullable TimestampSpec timestampSpec) + { + this.timestampSpec = timestampSpec; + return this; + } + + public Builder queryGranularity(@Nullable Granularity queryGranularity) + { + this.queryGranularity = queryGranularity; + return this; + } + public SegmentAnalysis build() { return new SegmentAnalysis( segmentId, intervals, columns, - size.orElse(0), - numRows.orElse(0), + size.orElse(0L), + numRows.orElse(0L), aggregators.isEmpty() ? null : aggregators, projections.isEmpty() ? null : projections, - null, - null, - rollup.orElse(null) + timestampSpec, + queryGranularity, + rollup.orElse(null), + containers.isEmpty() ? null : containers ); } } + + /** + * On-disk byte size of a single segment file container (a V10 file format bundle), reported by + * {@link SegmentMetadataQuery.AnalysisType#CONTAINERSIZE}. One entry per physical container; a bundle (e.g. a + * projection name) may have more than one container if its contents exceeded the writer's max container size. + * + * @param bundle owning bundle name (e.g. the base table or a projection's name) + * @param size on-disk byte size of this container + */ + public record ContainerAnalysis( + @JsonProperty("bundle") String bundle, + @JsonProperty("size") long size + ) + { + } } diff --git a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java index bf407e0cc9ac..d31a7bdca55d 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java @@ -60,7 +60,14 @@ public enum AnalysisType implements Cacheable TIMESTAMPSPEC, QUERYGRANULARITY, ROLLUP, - PROJECTIONS; + PROJECTIONS, + /** + * Reports {@link SegmentAnalysis#getContainers()}: per-container on-disk byte sizes. Only populated for segments + * written in the V10 file format ({@code IndexMergerV10}); pre-V10 segments report {@code null} here, same as + * any other analysis type applied to a segment that predates it. Only counts containers in the segment's + * entry-point file; a bundle whose data spilled into an attached external file is undercounted. + */ + CONTAINERSIZE; @JsonValue @Override @@ -198,6 +205,11 @@ public boolean hasProjections() return analysisTypes.contains(AnalysisType.PROJECTIONS); } + public boolean hasContainerSizes() + { + return analysisTypes.contains(AnalysisType.CONTAINERSIZE); + } + public boolean hasTimestampSpec() { return analysisTypes.contains(AnalysisType.TIMESTAMPSPEC); diff --git a/processing/src/main/java/org/apache/druid/segment/PartialQueryableIndex.java b/processing/src/main/java/org/apache/druid/segment/PartialQueryableIndex.java index e36d5916d0e3..adee27f970b0 100644 --- a/processing/src/main/java/org/apache/druid/segment/PartialQueryableIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/PartialQueryableIndex.java @@ -40,6 +40,7 @@ import org.apache.druid.segment.data.Indexed; import org.apache.druid.segment.data.ListIndexed; import org.apache.druid.segment.file.PartialSegmentFileMapperV10; +import org.apache.druid.segment.file.SegmentFileContainerMetadata; import org.apache.druid.segment.file.SegmentFileMapper; import org.apache.druid.segment.file.SegmentFileMetadata; import org.apache.druid.segment.projections.AggregateProjectionSchema; @@ -298,6 +299,18 @@ public List getOrdering() return ordering; } + /** + * Doesn't include containers from any external mapper {@link #fileMapper} may have attached (see + * {@link PartialSegmentFileMapperV10}) — those aren't reflected in {@link #metadata}, so a bundle whose data + * spilled into an external file will be undercounted here. + */ + @Nullable + @Override + public List getFileContainers() + { + return metadata.getContainers(); + } + @Nullable @Override public BaseColumnHolder getColumnHolder(String columnName) diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndex.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndex.java index a7c2c04d7714..393a6554b74c 100644 --- a/processing/src/main/java/org/apache/druid/segment/QueryableIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndex.java @@ -25,6 +25,7 @@ import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.data.Indexed; +import org.apache.druid.segment.file.SegmentFileContainerMetadata; import org.apache.druid.segment.projections.ClusteredValueGroupsBaseTableSchema; import org.apache.druid.segment.projections.QueryableProjection; import org.apache.druid.segment.projections.TableClusterGroupSpec; @@ -147,4 +148,17 @@ default QueryableIndex getClusterGroupQueryableIndex(TableClusterGroupSpec group { return null; } + + /** + * Returns the on-disk containers backing this index (V10 file format bundles), or {@code null} if unknown or + * unsupported (e.g. legacy pre-V10 segments or in-memory indexes). Used by the {@code CONTAINERSIZE} segment + * metadata analysis type to report each container's owning bundle name and byte size. Implementations only report + * containers from the entry-point segment file; a bundle whose data spilled into an attached external file is + * undercounted, since the external file's own containers aren't included. + */ + @Nullable + default List getFileContainers() + { + return null; + } } diff --git a/processing/src/main/java/org/apache/druid/segment/SimpleQueryableIndex.java b/processing/src/main/java/org/apache/druid/segment/SimpleQueryableIndex.java index 065a64932326..878a115daaaf 100644 --- a/processing/src/main/java/org/apache/druid/segment/SimpleQueryableIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/SimpleQueryableIndex.java @@ -40,7 +40,10 @@ import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.data.Indexed; import org.apache.druid.segment.data.ListIndexed; +import org.apache.druid.segment.file.SegmentFileContainerMetadata; import org.apache.druid.segment.file.SegmentFileMapper; +import org.apache.druid.segment.file.SegmentFileMapperV10; +import org.apache.druid.segment.file.SegmentFileMetadata; import org.apache.druid.segment.projections.ClusteredValueGroupsBaseTableSchema; import org.apache.druid.segment.projections.Projections; import org.apache.druid.segment.projections.QueryableProjection; @@ -333,12 +336,23 @@ public SegmentFileMapper getFileMapper() return fileMapper; } + /** + * Doesn't include containers from any external mapper {@link #fileMapper} may have attached (see + * {@link SegmentFileMapperV10}) — those aren't reflected in {@link SegmentFileMetadata#getContainers()} of the + * entry-point mapper, so a bundle whose data spilled into an external file will be undercounted here. + */ + @Override + @Nullable + public List getFileContainers() + { + final SegmentFileMetadata segmentFileMetadata = fileMapper.getSegmentFileMetadata(); + return segmentFileMetadata == null ? null : segmentFileMetadata.getContainers(); + } + @Override public void close() { - if (fileMapper != null) { - fileMapper.close(); - } + fileMapper.close(); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/file/PartialSegmentFileMapperV10.java b/processing/src/main/java/org/apache/druid/segment/file/PartialSegmentFileMapperV10.java index df571ff26478..fedb87383843 100644 --- a/processing/src/main/java/org/apache/druid/segment/file/PartialSegmentFileMapperV10.java +++ b/processing/src/main/java/org/apache/druid/segment/file/PartialSegmentFileMapperV10.java @@ -378,6 +378,7 @@ private PartialSegmentFileMapperV10( this.bitmapLock = new ReentrantLock(); } + @Override public SegmentFileMetadata getSegmentFileMetadata() { return metadata; diff --git a/processing/src/main/java/org/apache/druid/segment/file/SegmentFileMapper.java b/processing/src/main/java/org/apache/druid/segment/file/SegmentFileMapper.java index b547968cd00d..358acfb65923 100644 --- a/processing/src/main/java/org/apache/druid/segment/file/SegmentFileMapper.java +++ b/processing/src/main/java/org/apache/druid/segment/file/SegmentFileMapper.java @@ -61,6 +61,13 @@ default ByteBuffer mapExternalFile(String filename, String name) throws IOExcept return mapFile(name); } + /** + * Returns the {@link SegmentFileMetadata} describing this mapper's containers and internal files, or {@code null} + * if unsupported (e.g. legacy pre-V10 mappers that don't track this structure). + */ + @Nullable + SegmentFileMetadata getSegmentFileMetadata(); + @Override void close(); } diff --git a/processing/src/main/java/org/apache/druid/segment/file/SegmentFileMapperV10.java b/processing/src/main/java/org/apache/druid/segment/file/SegmentFileMapperV10.java index 042ab5ba084b..83bbce473c88 100644 --- a/processing/src/main/java/org/apache/druid/segment/file/SegmentFileMapperV10.java +++ b/processing/src/main/java/org/apache/druid/segment/file/SegmentFileMapperV10.java @@ -157,6 +157,7 @@ public SegmentFileMapperV10( this.externalMappers = externalMappers; } + @Override public SegmentFileMetadata getSegmentFileMetadata() { return segmentFileMetadata; diff --git a/processing/src/test/java/org/apache/druid/query/DoubleStorageTest.java b/processing/src/test/java/org/apache/druid/query/DoubleStorageTest.java index 231ac5abbe38..aea6ec6ec873 100644 --- a/processing/src/test/java/org/apache/druid/query/DoubleStorageTest.java +++ b/processing/src/test/java/org/apache/druid/query/DoubleStorageTest.java @@ -145,108 +145,101 @@ public DoubleStorageTest( @Parameterized.Parameters public static Collection dataFeeder() { - SegmentAnalysis expectedSegmentAnalysisDouble = new SegmentAnalysis( - SEGMENT_ID.toString(), - ImmutableList.of(INTERVAL), - new LinkedHashMap<>( - ImmutableMap.of( - TIME_COLUMN, - new ColumnAnalysis( - ColumnType.LONG, - ValueType.LONG.name(), - false, - false, - 100, - null, - null, - null, - null - ), - DIM_NAME, - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.name(), - false, - false, - 120, - 1, - DIM_VALUE, - DIM_VALUE, - null - ), - DIM_FLOAT_NAME, - new ColumnAnalysis( - ColumnType.DOUBLE, - ValueType.DOUBLE.name(), - false, - false, - 80, - null, - null, - null, - null + SegmentAnalysis expectedSegmentAnalysisDouble = new SegmentAnalysis.Builder(SEGMENT_ID) + .interval(INTERVAL) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + TIME_COLUMN, + new ColumnAnalysis( + ColumnType.LONG, + ValueType.LONG.name(), + false, + false, + 100, + null, + null, + null, + null + ), + DIM_NAME, + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.name(), + false, + false, + 120, + 1, + DIM_VALUE, + DIM_VALUE, + null + ), + DIM_FLOAT_NAME, + new ColumnAnalysis( + ColumnType.DOUBLE, + ValueType.DOUBLE.name(), + false, + false, + 80, + null, + null, + null, + null + ) ) ) - ), 330, - MAX_ROWS, - null, - null, - null, - null, - null - ); + ) + .size(330) + .numRows(MAX_ROWS) + .build(); - SegmentAnalysis expectedSegmentAnalysisFloat = new SegmentAnalysis( - SEGMENT_ID.toString(), - ImmutableList.of(INTERVAL), - new LinkedHashMap<>( - ImmutableMap.of( - TIME_COLUMN, - new ColumnAnalysis( - ColumnType.LONG, - ValueType.LONG.name(), - false, - false, - 100, - null, - null, - null, - null - ), - DIM_NAME, - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.name(), - false, - false, - 120, - 1, - DIM_VALUE, - DIM_VALUE, - null - ), - DIM_FLOAT_NAME, - new ColumnAnalysis( - ColumnType.FLOAT, - ValueType.FLOAT.name(), - false, - false, - 80, - null, - null, - null, - null + SegmentAnalysis expectedSegmentAnalysisFloat = new SegmentAnalysis.Builder(SEGMENT_ID) + .interval(INTERVAL) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + TIME_COLUMN, + new ColumnAnalysis( + ColumnType.LONG, + ValueType.LONG.name(), + false, + false, + 100, + null, + null, + null, + null + ), + DIM_NAME, + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.name(), + false, + false, + 120, + 1, + DIM_VALUE, + DIM_VALUE, + null + ), + DIM_FLOAT_NAME, + new ColumnAnalysis( + ColumnType.FLOAT, + ValueType.FLOAT.name(), + false, + false, + 80, + null, + null, + null, + null + ) ) ) - ), - 330, - MAX_ROWS, - null, - null, - null, - null, - null - ); + ) + .size(330) + .numRows(MAX_ROWS) + .build(); return ImmutableList.of( new Object[]{"double", expectedSegmentAnalysisDouble}, diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java index 28e2c91dabc6..a3865edfcc02 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java @@ -30,6 +30,7 @@ import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentAnalysis.ContainerAnalysis; import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.column.ColumnType; @@ -73,14 +74,13 @@ public void testSerde() throws Exception new ColumnAnalysis(ColumnType.DOUBLE, ColumnType.DOUBLE.asTypeString(), true, true, 0, null, null, null, null) ); - final SegmentAnalysis analysis = new SegmentAnalysis( - "id", - Intervals.ONLY_ETERNITY, - columns, - 1, - 2, - ImmutableMap.of("cnt", new CountAggregatorFactory("cnt")), - ImmutableMap.of("channel_added_hourly", new AggregateProjectionMetadata( + final SegmentAnalysis analysis = new SegmentAnalysis.Builder("id") + .intervals(Intervals.ONLY_ETERNITY) + .columns(columns) + .size(1) + .numRows(2) + .aggregators(ImmutableMap.of("cnt", new CountAggregatorFactory("cnt"))) + .projections(ImmutableMap.of("channel_added_hourly", new AggregateProjectionMetadata( AggregateProjectionSchema.schemaBuilder("channel_added_hourly") .timeColumnName(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) .virtualColumns( @@ -93,11 +93,15 @@ public void testSerde() throws Exception .aggregators(new LongSumAggregatorFactory("sum_added", "added")) .build(), 16 - )), - TimestampSpec.DEFAULT, - Granularities.SECOND, - true - ); + ))) + .timestampSpec(TimestampSpec.DEFAULT) + .queryGranularity(Granularities.SECOND) + .rollup(true) + .containers(ImmutableList.of( + new ContainerAnalysis("__base", 223), + new ContainerAnalysis("channel_added_hourly", 145) + )) + .build(); final ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); final SegmentAnalysis analysis2 = jsonMapper.readValue( @@ -112,5 +116,8 @@ public void testSerde() throws Exception ImmutableList.copyOf(columns.entrySet()), ImmutableList.copyOf(analysis2.getColumns().entrySet()) ); + + // Verify containers survive serde. + Assertions.assertEquals(analysis.getContainers(), analysis2.getContainers()); } } diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryContainerSizeTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryContainerSizeTest.java new file mode 100644 index 000000000000..168495ed68e9 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryContainerSizeTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.metadata; + +import com.google.common.collect.ImmutableMap; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.MapBasedInputRow; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.FileUtils; +import org.apache.druid.query.Druids; +import org.apache.druid.query.QueryPlus; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryRunnerTestHelper; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.metadata.metadata.SegmentAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentAnalysis.ContainerAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.segment.IndexBuilder; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.QueryableIndexSegment; +import org.apache.druid.segment.incremental.IncrementalIndexSchema; +import org.apache.druid.segment.projections.Projections; +import org.apache.druid.timeline.SegmentId; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.util.List; + +/** + * Verifies {@link SegmentMetadataQuery.AnalysisType#CONTAINERSIZE} against a real V10-format segment: containers are + * only populated when the analysis type is requested, and their bundle names line up with the base table and + * projection names used at write time. + */ +public class SegmentMetadataQueryContainerSizeTest +{ + private static final String PROJECTION_NAME = "dim_sum"; + private static final String DATASOURCE = "containerSizeTestDatasource"; + + private static final SegmentMetadataQueryRunnerFactory FACTORY = new SegmentMetadataQueryRunnerFactory( + new SegmentMetadataQueryQueryToolChest(new SegmentMetadataQueryConfig()), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); + + @Test + public void testContainerSizeAnalysis() + { + final AggregateProjectionSpec projectionSpec = + AggregateProjectionSpec.builder(PROJECTION_NAME) + .groupingColumns(new StringDimensionSchema("dim")) + .aggregators(new LongSumAggregatorFactory("m_sum", "m_sum")) + .build(); + + final IncrementalIndexSchema schema = + IncrementalIndexSchema.builder() + .withDimensionsSpec(new DimensionsSpec(List.of(new StringDimensionSchema("dim")))) + .withMetrics(new LongSumAggregatorFactory("m_sum", "m")) + .withRollup(false) + .withMinTimestamp(DateTimes.of("2013-01-01").getMillis()) + .withProjections(List.of(projectionSpec)) + .build(); + + final List rows = List.of( + new MapBasedInputRow(DateTimes.of("2013-01-01"), List.of("dim"), ImmutableMap.of("dim", "a", "m", 1L)), + new MapBasedInputRow(DateTimes.of("2013-01-01"), List.of("dim"), ImmutableMap.of("dim", "b", "m", 2L)) + ); + + final File tmpDir = FileUtils.createTempDir(); + final QueryableIndex index = IndexBuilder.create() + .useV10() + .tmpDir(tmpDir) + .schema(schema) + .rows(rows) + .buildMMappedIndex(); + + final SegmentId segmentId = SegmentId.dummy(DATASOURCE); + final QueryRunner runner = QueryRunnerTestHelper.makeQueryRunner( + FACTORY, + segmentId, + new QueryableIndexSegment(index, segmentId), + null + ); + + final SegmentMetadataQuery query = + Druids.newSegmentMetadataQueryBuilder() + .dataSource(DATASOURCE) + .intervals("2013/2014") + .analysisTypes(SegmentMetadataQuery.AnalysisType.CONTAINERSIZE) + .merge(false) + .build(); + + final List results = runner.run(QueryPlus.wrap(query)).toList(); + Assert.assertEquals(1, results.size()); + + final List containers = results.get(0).getContainers(); + Assert.assertNotNull(containers); + Assert.assertEquals(2, containers.size()); + Assert.assertEquals(Projections.BASE_TABLE_PROJECTION_NAME, containers.get(0).bundle()); + Assert.assertEquals(PROJECTION_NAME, containers.get(1).bundle()); + + final long baseSize = containers.get(0).size(); + final long projectionSize = containers.get(1).size(); + // loose upper bound: 2 rows of data should never legitimately serialize to this many bytes; this only catches + // gross errors (e.g. reporting an offset or a whole-file size instead of the container's own size). + Assert.assertTrue("projection container size should be positive", projectionSize > 0); + // the base table has an extra __time column that the (time-less) projection doesn't, so it should be larger. + Assert.assertTrue( + "base table container should be larger than the projection's (extra __time column)", + baseSize > projectionSize + ); + Assert.assertTrue("base table container size looks implausibly large", baseSize < 10_000); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java index df0005c69353..90675900ef56 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java @@ -675,6 +675,55 @@ public void testProjectionsWithConflict(AggregatorMergeStrategy aggregatorMergeS Assert.assertEquals(expectedStrict, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); } + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testContainers(AggregatorMergeStrategy aggregatorMergeStrategy) + { + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1) + .container("__base", 100) + .container("channel_sum", 50) + .build(); + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2) + .container("__base", 200) + .container("channel_sum", 75) + .build(); + + // merging sums sizes per bundle name, rather than concatenating each segment's raw container list. + final SegmentAnalysis expected = new SegmentAnalysis.Builder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .container("__base", 300) + .container("channel_sum", 125) + .build(); + Assert.assertEquals(expected, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); + } + + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testContainersOneSided(AggregatorMergeStrategy aggregatorMergeStrategy) + { + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1) + .container("__base", 100) + .build(); + final SegmentAnalysis analysis2NoContainers = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2).build(); + + // a segment with no container info contributes nothing, rather than nulling out the whole merged result. + final SegmentAnalysis expected = new SegmentAnalysis.Builder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .container("__base", 100) + .build(); + Assert.assertEquals(expected, mergeWithStrategy(analysis1, analysis2NoContainers, aggregatorMergeStrategy)); + } + + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testContainersBothNull(AggregatorMergeStrategy aggregatorMergeStrategy) + { + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2).build(); + + Assert.assertNull(mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy).getContainers()); + } + private static SegmentAnalysis mergeWithStrategy( SegmentAnalysis analysis1, SegmentAnalysis analysis2, diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java index 780437edfb2c..1e75adda19d2 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java @@ -235,108 +235,104 @@ public SegmentMetadataQueryTest( new AggregateProjectionMetadata(PROJECTION2_SCHEMA, PROJECTION2_ROWS) ); - expectedSegmentAnalysis1 = new SegmentAnalysis( - id1.toString(), - ImmutableList.of(Intervals.of("2011-01-12T00:00:00.000Z/2011-04-15T00:00:00.001Z")), - new LinkedHashMap<>( - ImmutableMap.of( - "__time", - new ColumnAnalysis( - ColumnType.LONG, - ValueType.LONG.toString(), - false, - false, - 12090, - null, - null, - null, - null - ), - "index", - new ColumnAnalysis( - ColumnType.DOUBLE, - ValueType.DOUBLE.toString(), - false, - false, - 9672, - null, - null, - null, - null - ), - "placement", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - false, - false, - placementSize, - 1, - "preferred", - "preferred", - null + expectedSegmentAnalysis1 = new SegmentAnalysis.Builder(id1) + .interval(Intervals.of("2011-01-12T00:00:00.000Z/2011-04-15T00:00:00.001Z")) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + "__time", + new ColumnAnalysis( + ColumnType.LONG, + ValueType.LONG.toString(), + false, + false, + 12090, + null, + null, + null, + null + ), + "index", + new ColumnAnalysis( + ColumnType.DOUBLE, + ValueType.DOUBLE.toString(), + false, + false, + 9672, + null, + null, + null, + null + ), + "placement", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + false, + false, + placementSize, + 1, + "preferred", + "preferred", + null + ) ) ) - ), - overallSize, - 1209, - expectedAggregators, - expectedProjections, - null, - null, - null - ); - expectedSegmentAnalysis2 = new SegmentAnalysis( - id2.toString(), - ImmutableList.of(Intervals.of("2011-01-12T00:00:00.000Z/2011-04-15T00:00:00.001Z")), - new LinkedHashMap<>( - ImmutableMap.of( - "__time", - new ColumnAnalysis( - ColumnType.LONG, - ValueType.LONG.toString(), - false, - false, - 12090, - null, - null, - null, - null - ), - "index", - new ColumnAnalysis( - ColumnType.DOUBLE, - ValueType.DOUBLE.toString(), - false, - false, - 9672, - null, - null, - null, - null - ), - "placement", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - false, - false, - placementSize, - 1, - null, - null, - null + ) + .size(overallSize) + .numRows(1209) + .aggregators(expectedAggregators) + .projections(expectedProjections) + .build(); + expectedSegmentAnalysis2 = new SegmentAnalysis.Builder(id2) + .interval(Intervals.of("2011-01-12T00:00:00.000Z/2011-04-15T00:00:00.001Z")) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + "__time", + new ColumnAnalysis( + ColumnType.LONG, + ValueType.LONG.toString(), + false, + false, + 12090, + null, + null, + null, + null + ), + "index", + new ColumnAnalysis( + ColumnType.DOUBLE, + ValueType.DOUBLE.toString(), + false, + false, + 9672, + null, + null, + null, + null + ), + "placement", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + false, + false, + placementSize, + 1, + null, + null, + null + ) ) ) - ), - overallSize, - 1209, - expectedAggregators, - expectedProjections, - null, - null, - null - ); + ) + .size(overallSize) + .numRows(1209) + .aggregators(expectedAggregators) + .projections(expectedProjections) + .build(); } @Test @@ -348,6 +344,7 @@ public void testSegmentMetadataQuery() Assert.assertEquals(Collections.singletonList(expectedSegmentAnalysis1), results); } + @Test public void testSegmentMetadataQueryOnRestricted() { @@ -378,45 +375,42 @@ public void testSegmentMetadataQueryOnUnion() @Test public void testSegmentMetadataQueryWithRollupMerge() { - SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis( - differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString(), - null, - new LinkedHashMap<>( - ImmutableMap.of( - "placement", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - false, - false, - 0, - 0, - null, - null, - null - ), - "placementish", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - true, - false, - 0, - 0, - null, - null, - null + SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis.Builder( + differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString()) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + "placement", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + false, + false, + 0, + 0, + null, + null, + null + ), + "placementish", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + true, + false, + 0, + 0, + null, + null, + null + ) ) ) - ), - 0, - expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), - null, - null, - null, - null, - rollup1 != rollup2 ? null : rollup1 - ); + ) + .size(0) + .numRows(expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows()) + .rollup(rollup1 != rollup2 ? null : rollup1) + .build(); QueryToolChest toolChest = FACTORY.getToolchest(); @@ -453,45 +447,41 @@ public void testSegmentMetadataQueryWithRollupMerge() @Test public void testSegmentMetadataQueryWithHasMultipleValuesMerge() { - SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis( - differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString(), - null, - new LinkedHashMap<>( - ImmutableMap.of( - "placement", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - false, - false, - 0, - 1, - null, - null, - null - ), - "placementish", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - true, - false, - 0, - 9, - null, - null, - null + SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis.Builder( + differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString()) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + "placement", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + false, + false, + 0, + 1, + null, + null, + null + ), + "placementish", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + true, + false, + 0, + 9, + null, + null, + null + ) ) ) - ), - 0, - expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), - null, - null, - null, - null, - null - ); + ) + .size(0) + .numRows(expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows()) + .build(); QueryToolChest toolChest = FACTORY.getToolchest(); @@ -528,45 +518,41 @@ public void testSegmentMetadataQueryWithHasMultipleValuesMerge() @Test public void testSegmentMetadataQueryWithComplexColumnMerge() { - SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis( - differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString(), - null, - new LinkedHashMap<>( - ImmutableMap.of( - "placement", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - false, - false, - 0, - 1, - null, - null, - null - ), - "quality_uniques", - new ColumnAnalysis( - ColumnType.ofComplex("hyperUnique"), - "hyperUnique", - false, - true, - 0, - null, - null, - null, - null + SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis.Builder( + differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString()) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + "placement", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + false, + false, + 0, + 1, + null, + null, + null + ), + "quality_uniques", + new ColumnAnalysis( + ColumnType.ofComplex("hyperUnique"), + "hyperUnique", + false, + true, + 0, + null, + null, + null, + null + ) ) ) - ), - 0, - expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), - null, - null, - null, - null, - null - ); + ) + .size(0) + .numRows(expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows()) + .build(); QueryToolChest toolChest = FACTORY.getToolchest(); @@ -673,52 +659,53 @@ private void testSegmentMetadataQueryWithDefaultAnalysisMerge( expectedAggregators.put(agg.getName(), agg.getCombiningFactory()); } - SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis( - differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString(), - ImmutableList.of(expectedSegmentAnalysis1.getIntervals().get(0)), - new LinkedHashMap<>( + SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis.Builder( + differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString()) + .interval(expectedSegmentAnalysis1.getIntervals().get(0)) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + "__time", + new ColumnAnalysis( + ColumnType.LONG, + ValueType.LONG.toString(), + false, + false, + 12090 * 2, + null, + null, + null, + null + ), + "index", + new ColumnAnalysis( + ColumnType.DOUBLE, + ValueType.DOUBLE.toString(), + false, + false, + 9672 * 2, + null, + null, + null, + null + ), + column, + analysis + ) + ) + ) + .size(expectedSegmentAnalysis1.getSize() + expectedSegmentAnalysis2.getSize()) + .numRows(expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows()) + .aggregators(expectedAggregators) + .projections( ImmutableMap.of( - "__time", - new ColumnAnalysis( - ColumnType.LONG, - ValueType.LONG.toString(), - false, - false, - 12090 * 2, - null, - null, - null, - null - ), - "index", - new ColumnAnalysis( - ColumnType.DOUBLE, - ValueType.DOUBLE.toString(), - false, - false, - 9672 * 2, - null, - null, - null, - null - ), - column, - analysis + PROJECTION1_SCHEMA.getName(), + new AggregateProjectionMetadata(PROJECTION1_SCHEMA, PROJECTION1_ROWS * 2), + PROJECTION2_SCHEMA.getName(), + new AggregateProjectionMetadata(PROJECTION2_SCHEMA, PROJECTION2_ROWS * 2) ) - ), - expectedSegmentAnalysis1.getSize() + expectedSegmentAnalysis2.getSize(), - expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), - expectedAggregators, - ImmutableMap.of( - PROJECTION1_SCHEMA.getName(), - new AggregateProjectionMetadata(PROJECTION1_SCHEMA, PROJECTION1_ROWS * 2), - PROJECTION2_SCHEMA.getName(), - new AggregateProjectionMetadata(PROJECTION2_SCHEMA, PROJECTION2_ROWS * 2) - ), - null, - null, - null - ); + ) + .build(); QueryToolChest toolChest = FACTORY.getToolchest(); @@ -749,33 +736,29 @@ private void testSegmentMetadataQueryWithDefaultAnalysisMerge( @Test public void testSegmentMetadataQueryWithNoAnalysisTypesMerge() { - SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis( - differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString(), - null, - new LinkedHashMap<>( - ImmutableMap.of( - "placement", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - false, - false, - 0, - 0, - null, - null, - null + SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis.Builder( + differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString()) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + "placement", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + false, + false, + 0, + 0, + null, + null, + null + ) ) ) - ), - 0, - expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), - null, - null, - null, - null, - null - ); + ) + .size(0) + .numRows(expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows()) + .build(); QueryToolChest toolChest = FACTORY.getToolchest(); @@ -816,33 +799,30 @@ public void testSegmentMetadataQueryWithAggregatorsMerge() for (AggregatorFactory agg : TestIndex.METRIC_AGGS) { expectedAggregators.put(agg.getName(), agg.getCombiningFactory()); } - SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis( - differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString(), - null, - new LinkedHashMap<>( - ImmutableMap.of( - "placement", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - false, - false, - 0, - 0, - null, - null, - null + SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis.Builder( + differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString()) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + "placement", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + false, + false, + 0, + 0, + null, + null, + null + ) ) ) - ), - 0, - expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), - expectedAggregators, - null, - null, - null, - null - ); + ) + .size(0) + .numRows(expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows()) + .aggregators(expectedAggregators) + .build(); QueryToolChest toolChest = FACTORY.getToolchest(); @@ -883,33 +863,30 @@ public void testSegmentMetadataQueryWithAggregatorsMergeLenientStrategy() for (AggregatorFactory agg : TestIndex.METRIC_AGGS) { expectedAggregators.put(agg.getName(), agg.getCombiningFactory()); } - SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis( - differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString(), - null, - new LinkedHashMap<>( - ImmutableMap.of( - "placement", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - false, - false, - 0, - 0, - null, - null, - null + SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis.Builder( + differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString()) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + "placement", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + false, + false, + 0, + 0, + null, + null, + null + ) ) ) - ), - 0, - expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), - expectedAggregators, - null, - null, - null, - null - ); + ) + .size(0) + .numRows(expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows()) + .aggregators(expectedAggregators) + .build(); QueryToolChest toolChest = FACTORY.getToolchest(); @@ -947,33 +924,30 @@ public void testSegmentMetadataQueryWithAggregatorsMergeLenientStrategy() @Test public void testSegmentMetadataQueryWithTimestampSpecMerge() { - SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis( - differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString(), - null, - new LinkedHashMap<>( - ImmutableMap.of( - "placement", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - false, - false, - 0, - 0, - null, - null, - null + SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis.Builder( + differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString()) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + "placement", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + false, + false, + 0, + 0, + null, + null, + null + ) ) ) - ), - 0, - expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), - null, - null, - new TimestampSpec("ts", "iso", null), - null, - null - ); + ) + .size(0) + .numRows(expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows()) + .timestampSpec(new TimestampSpec("ts", "iso", null)) + .build(); QueryToolChest toolChest = FACTORY.getToolchest(); @@ -1010,33 +984,30 @@ public void testSegmentMetadataQueryWithTimestampSpecMerge() @Test public void testSegmentMetadataQueryWithQueryGranularityMerge() { - SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis( - differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString(), - null, - new LinkedHashMap<>( - ImmutableMap.of( - "placement", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - false, - false, - 0, - 0, - null, - null, - null + SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis.Builder( + differentIds ? "merged" : SegmentId.dummy(DATASOURCE).toString()) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + "placement", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + false, + false, + 0, + 0, + null, + null, + null + ) ) ) - ), - 0, - expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), - null, - null, - null, - Granularities.NONE, - null - ); + ) + .size(0) + .numRows(expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows()) + .queryGranularity(Granularities.NONE) + .build(); QueryToolChest toolChest = FACTORY.getToolchest(); diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java index 7082dd744ed4..1bf373ec375c 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java @@ -98,33 +98,29 @@ public static Iterable constructorFeeder() @Test public void testSegmentMetadataUnionQuery() { - SegmentAnalysis expected = new SegmentAnalysis( - QueryRunnerTestHelper.SEGMENT_ID.toString(), - Collections.singletonList(Intervals.of("2011-01-12T00:00:00.000Z/2011-04-15T00:00:00.001Z")), - new LinkedHashMap<>( - ImmutableMap.of( - "placement", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.toString(), - false, - false, - 43524, - 1, - "preferred", - "preferred", - null + SegmentAnalysis expected = new SegmentAnalysis.Builder(QueryRunnerTestHelper.SEGMENT_ID) + .interval(Intervals.of("2011-01-12T00:00:00.000Z/2011-04-15T00:00:00.001Z")) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( + "placement", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.toString(), + false, + false, + 43524, + 1, + "preferred", + "preferred", + null + ) ) ) - ), - 805380, - 4836, - null, - null, - null, - null, - null - ); + ) + .size(805380) + .numRows(4836) + .build(); SegmentMetadataQuery query = new Druids.SegmentMetadataQueryBuilder() .dataSource(QueryRunnerTestHelper.UNION_DATA_SOURCE) .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC) diff --git a/processing/src/test/java/org/apache/druid/segment/SimpleQueryableIndexClusteredTest.java b/processing/src/test/java/org/apache/druid/segment/SimpleQueryableIndexClusteredTest.java index 8d46f0884b81..f89f15dde6f3 100644 --- a/processing/src/test/java/org/apache/druid/segment/SimpleQueryableIndexClusteredTest.java +++ b/processing/src/test/java/org/apache/druid/segment/SimpleQueryableIndexClusteredTest.java @@ -28,6 +28,7 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.data.ListIndexed; +import org.apache.druid.segment.file.NoopSegmentFileMapper; import org.apache.druid.segment.projections.ClusterGroupSchemaTestHelpers; import org.apache.druid.segment.projections.ClusteredValueGroupsBaseTableSchema; import org.apache.druid.segment.projections.TableClusterGroupSpec; @@ -93,7 +94,7 @@ private static SimpleQueryableIndex buildClusteredIndex( new ListIndexed<>(List.of()), new RoaringBitmapFactory(), Map.of(), // clustered summary has no top-level columns - null, // no SegmentFileMapper for in-memory test + NoopSegmentFileMapper.INSTANCE, // no real SegmentFileMapper for in-memory test reconstructed, projectionColumns, summary, @@ -152,7 +153,7 @@ void testNonClusteredIndexHasNullSummaryAndEmptyGroups() new ListIndexed<>(List.of()), new RoaringBitmapFactory(), Map.of(), - null, + NoopSegmentFileMapper.INSTANCE, null, null ) diff --git a/processing/src/test/java/org/apache/druid/segment/file/NoopSegmentFileMapper.java b/processing/src/test/java/org/apache/druid/segment/file/NoopSegmentFileMapper.java new file mode 100644 index 000000000000..a7cc4b7fbde9 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/file/NoopSegmentFileMapper.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.file; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.Set; + +/** + * A {@link SegmentFileMapper} with no backing files, for tests that build a + * {@link org.apache.druid.segment.SimpleQueryableIndex} purely in-memory and have no real segment file. Use this + * instead of a null {@code SegmentFileMapper}, so that {@code fileMapper} is never null and methods like + * {@link org.apache.druid.segment.QueryableIndex#getFileContainers()} don't need to guard against it. + */ +public class NoopSegmentFileMapper implements SegmentFileMapper +{ + public static final NoopSegmentFileMapper INSTANCE = new NoopSegmentFileMapper(); + + private NoopSegmentFileMapper() + { + } + + @Override + public Set getInternalFilenames() + { + return Set.of(); + } + + @Nullable + @Override + public ByteBuffer mapFile(String name) + { + return null; + } + + @Nullable + @Override + public SegmentFileMetadata getSegmentFileMetadata() + { + return null; + } + + @Override + public void close() + { + } +} diff --git a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java index 3688b93f818e..1ad845bf2e37 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java @@ -1108,18 +1108,12 @@ public void testSegmentMetadataColumnType() ); RowSignature signature = AbstractSegmentMetadataCache.analysisToRowSignature( - new SegmentAnalysis( - "id", - ImmutableList.of(Intervals.utc(1L, 2L)), - columns, - 1234, - 100, - null, - null, - null, - null, - null - ) + new SegmentAnalysis.Builder("id") + .interval(Intervals.utc(1L, 2L)) + .columns(columns) + .size(1234) + .numRows(100) + .build() ); Assert.assertEquals( @@ -1136,57 +1130,53 @@ public void testSegmentMetadataColumnType() public void testSegmentMetadataFallbackType() { RowSignature signature = AbstractSegmentMetadataCache.analysisToRowSignature( - new SegmentAnalysis( - "id", - ImmutableList.of(Intervals.utc(1L, 2L)), - new LinkedHashMap<>( - ImmutableMap.of( - "a", - new ColumnAnalysis( - null, - ColumnType.STRING.asTypeString(), - false, - true, - 1234, - 26, + new SegmentAnalysis.Builder("id") + .interval(Intervals.utc(1L, 2L)) + .columns( + new LinkedHashMap<>( + ImmutableMap.of( "a", - "z", - null - ), - "count", - new ColumnAnalysis( - null, - ColumnType.LONG.asTypeString(), - false, - true, - 1234, - null, - null, - null, - null - ), - "distinct", - new ColumnAnalysis( - null, - "hyperUnique", - false, - true, - 1234, - null, - null, - null, - null + new ColumnAnalysis( + null, + ColumnType.STRING.asTypeString(), + false, + true, + 1234, + 26, + "a", + "z", + null + ), + "count", + new ColumnAnalysis( + null, + ColumnType.LONG.asTypeString(), + false, + true, + 1234, + null, + null, + null, + null + ), + "distinct", + new ColumnAnalysis( + null, + "hyperUnique", + false, + true, + 1234, + null, + null, + null, + null + ) ) ) - ), - 1234, - 100, - null, - null, - null, - null, - null - ) + ) + .size(1234) + .numRows(100) + .build() ); Assert.assertEquals( RowSignature.builder().add("a", ColumnType.STRING).add("count", ColumnType.LONG).add("distinct", ColumnType.ofComplex("hyperUnique")).build(), @@ -1211,18 +1201,12 @@ public void testAnalysisToRowSignatureDoesNotSkipColumnsWhenAnalysisHasErrors() columns.put("error_col2", ColumnAnalysis.error("multi_value")); final RowSignature signature = AbstractSegmentMetadataCache.analysisToRowSignature( - new SegmentAnalysis( - "id", - ImmutableList.of(Intervals.utc(1L, 2L)), - columns, - 1234, - 100, - null, - null, - null, - null, - null - ) + new SegmentAnalysis.Builder("id") + .interval(Intervals.utc(1L, 2L)) + .columns(columns) + .size(1234) + .numRows(100) + .build() ); Assert.assertEquals(