diff --git a/architecture-test/pom.xml b/architecture-test/pom.xml new file mode 100644 index 000000000000..6ba856b7df8c --- /dev/null +++ b/architecture-test/pom.xml @@ -0,0 +1,284 @@ + + + + 4.0.0 + + org.apache.iotdb + iotdb-parent + 2.0.11-SNAPSHOT + + iotdb-architecture-test + IoTDB: Repository Architecture Tests + + + org.apache.tsfile + common + ${tsfile.version} + test + + + org.apache.tsfile + tsfile + test + + + com.tngtech.archunit + archunit + 1.3.0 + test + + + org.ow2.asm + asm + test + + + junit + junit + test + + + org.apache.iotdb + udf-api + ${project.version} + test + + + org.apache.iotdb + external-service-api + ${project.version} + test + + + org.apache.iotdb + external-api + ${project.version} + test + + + org.apache.iotdb + pipe-api + ${project.version} + test + + + org.apache.iotdb + trigger-api + ${project.version} + test + + + org.apache.iotdb + iotdb-cli + ${project.version} + test + + + org.apache.iotdb + iotdb-jdbc + ${project.version} + test + + + org.apache.iotdb + iotdb-session + ${project.version} + test + + + org.apache.iotdb + isession + ${project.version} + test + + + org.apache.iotdb + service-rpc + ${project.version} + test + + + org.apache.iotdb + iotdb-subscription + ${project.version} + test + + + org.apache.iotdb + library-udf + ${project.version} + test + + + org.apache.iotdb + iotdb-relational-grammar + ${project.version} + test + + + org.apache.iotdb + node-commons + ${project.version} + test + + + org.apache.iotdb + calc-commons + ${project.version} + test + + + org.apache.iotdb + metrics-interface + ${project.version} + test + + + org.apache.iotdb + metrics-core + ${project.version} + test + + + org.apache.iotdb + iotdb-consensus + ${project.version} + test + + + org.apache.iotdb + iotdb-confignode + ${project.version} + test + + + org.apache.iotdb + iotdb-server + ${project.version} + test + + + org.apache.iotdb + rest + ${project.version} + test + + + org.apache.iotdb + rest-openapi + ${project.version} + test + + + org.apache.iotdb + mqtt + ${project.version} + test + + + org.apache.iotdb + udf-example + ${project.version} + test + + + org.apache.iotdb + trigger-example + ${project.version} + test + + + org.apache.iotdb + subscription-example + ${project.version} + test + + + org.apache.iotdb + client-example + ${project.version} + test + + + org.apache.iotdb + schema-example + ${project.version} + test + + + org.apache.iotdb + customize-mqtt-example + ${project.version} + test + + + org.apache.iotdb + mqtt-example + ${project.version} + test + + + org.apache.iotdb + pipe-opc-ua-sink-example + ${project.version} + test + + + org.apache.iotdb + pipe-count-point-processor-example + ${project.version} + test + + + org.apache.iotdb + jdbc-example + ${project.version} + test + + + org.apache.iotdb + rest-java-example + ${project.version} + test + + + org.apache.iotdb + iotdb-antlr + ${project.version} + test + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + + org.apache.iotdb:* + + + + + + diff --git a/architecture-test/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchArchitectureTest.java b/architecture-test/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchArchitectureTest.java new file mode 100644 index 000000000000..be9cec253bb9 --- /dev/null +++ b/architecture-test/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchArchitectureTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import com.tngtech.archunit.core.importer.ClassFileImporter; +import com.tngtech.archunit.core.importer.ImportOption; +import org.junit.Test; + +public class TSDataTypeSwitchArchitectureTest { + + @Test + public void repositoryProductionCodeMustUseTypeService() { + // This aggregate module depends on every Java production module, so the scan is + // repository-wide. + TSDataTypeSwitchRule.RULE.check( + new ClassFileImporter() + .withImportOption(new ImportOption.DoNotIncludeTests()) + .withImportOption(location -> TSDataTypeSwitchRule.hasSwitch(location.asURI())) + .importPackages("org.apache.iotdb")); + } +} diff --git a/architecture-test/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchRule.java b/architecture-test/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchRule.java new file mode 100644 index 000000000000..6dad4d745e8b --- /dev/null +++ b/architecture-test/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchRule.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import com.tngtech.archunit.core.domain.JavaClass; +import com.tngtech.archunit.core.domain.JavaModifier; +import com.tngtech.archunit.lang.ArchCondition; +import com.tngtech.archunit.lang.ArchRule; +import com.tngtech.archunit.lang.ConditionEvents; +import com.tngtech.archunit.lang.SimpleConditionEvent; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.classes; + +final class TSDataTypeSwitchRule { + private static final String SWITCH_MAP = + "$SwitchMap$" + TSDataType.class.getName().replace('.', '$'); + private static final String SERVICE_DESCRIPTOR = Type.getDescriptor(TypeService.class); + + static final ArchRule RULE = + classes() + .should( + new ArchCondition<>("only switch on TSDataType inside TypeService") { + @Override + public void check(JavaClass javaClass, ConditionEvents events) { + if (javaClass.getModifiers().contains(JavaModifier.SYNTHETIC) + || javaClass.isAssignableTo(TypeService.class)) { + return; + } + SwitchVisitor visitor = read(javaClass.getSource().orElseThrow().getUri()); + for (SwitchAccess access : visitor.switches) { + if (!visitor.serviceMethods.contains(access.method)) { + events.add( + SimpleConditionEvent.violated( + javaClass, + javaClass.getName() + + "." + + access.method + + " switches on TSDataType outside TypeService at (" + + visitor.sourceFile + + ":" + + access.line + + ")")); + } + } + } + }) + .allowEmptyShould(true); + + private TSDataTypeSwitchRule() {} + + static boolean hasSwitch(URI uri) { + return !read(uri).switches.isEmpty(); + } + + private static SwitchVisitor read(URI uri) { + try (InputStream input = uri.toURL().openStream()) { + SwitchVisitor visitor = new SwitchVisitor(); + new ClassReader(input).accept(visitor, ClassReader.SKIP_FRAMES); + return visitor; + } catch (IOException e) { + throw new UncheckedIOException("Cannot inspect " + uri, e); + } + } + + // ArchUnit deliberately drops $SwitchMap$ accesses. Read them with ASM instead of treating + // every ordinal() invocation as a switch, which would incorrectly reject ordinary enum usage. + private static final class SwitchVisitor extends ClassVisitor { + private final List switches = new ArrayList<>(); + private final Set serviceMethods = new HashSet<>(); + private String owner; + private String sourceFile; + private boolean synthetic; + + private SwitchVisitor() { + super(Opcodes.ASM9); + } + + @Override + public void visit( + int version, + int access, + String name, + String signature, + String superName, + String[] interfaces) { + owner = name; + synthetic = (access & Opcodes.ACC_SYNTHETIC) != 0; + } + + @Override + public void visitSource(String source, String debug) { + sourceFile = source; + } + + @Override + public MethodVisitor visitMethod( + int access, String name, String descriptor, String signature, String[] exceptions) { + if (synthetic) { + // The compiler's mapping-array initializer is not a source-level switch. + return null; + } + return new MethodVisitor(Opcodes.ASM9) { + private int line; + + @Override + public void visitLineNumber(int lineNumber, Label start) { + line = lineNumber; + } + + @Override + public void visitFieldInsn( + int opcode, String fieldOwner, String fieldName, String fieldType) { + // The repository compiles with Java 17 javac, for both switch statements and expressions. + if (opcode == Opcodes.GETSTATIC + && fieldName.equals(SWITCH_MAP) + && fieldType.equals("[I")) { + switches.add(new SwitchAccess(name + descriptor, line)); + } + } + + @Override + public void visitInvokeDynamicInsn( + String name, String descriptor, Handle bootstrap, Object... arguments) { + // Only the lambda implementing TypeService is exempt, not all methods of its holder. + if (bootstrap.getOwner().equals("java/lang/invoke/LambdaMetafactory") + && Type.getReturnType(descriptor).getDescriptor().equals(SERVICE_DESCRIPTOR) + && arguments.length > 1 + && arguments[1] instanceof Handle implementation + && implementation.getOwner().equals(owner) + && implementation.getName().startsWith("lambda$")) { + serviceMethods.add(implementation.getName() + implementation.getDesc()); + } + } + }; + } + } + + private static final class SwitchAccess { + private final String method; + private final int line; + + private SwitchAccess(String method, int line) { + this.method = method; + this.line = line; + } + } +} diff --git a/architecture-test/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchRuleTest.java b/architecture-test/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchRuleTest.java new file mode 100644 index 000000000000..53a0f2b41084 --- /dev/null +++ b/architecture-test/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchRuleTest.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import com.tngtech.archunit.core.importer.ClassFileImporter; +import com.tngtech.archunit.lang.EvaluationResult; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.junit.Test; + +import java.util.List; +import java.util.function.ToIntFunction; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TSDataTypeSwitchRuleTest { + + @Test + public void detectsStatementAndExpression() { + // Both javac switch forms must report the business method and a nonzero source line. + List violations = violations(Switches.class); + assertEquals(2, violations.size()); + assertTrue(violations.stream().anyMatch(message -> message.contains(".statement("))); + assertTrue(violations.stream().anyMatch(message -> message.contains(".expression("))); + for (String message : violations) { + assertTrue(message, message.matches(".*TSDataTypeSwitchRuleTest.java:[1-9][0-9]*.*")); + } + } + + @Test + public void detectsLambdaAndAnonymousClass() { + // Switches cannot escape detection by being moved into lambdas or anonymous classes. + assertEquals(1, violations(LambdaSwitch.class).size()); + assertEquals(1, violations(AnonymousSwitch.VALUE.getClass()).size()); + } + + @Test + public void allowsOtherEnumsAndOrdinal() { + // An ordinal read beside an unrelated enum switch is legal, as is reading an enum constant. + assertTrue(violations(OrdinaryUsage.class).isEmpty()); + } + + @Test + public void allowsTypeServiceImplementationAndRegisteredHolder() { + // Actual TypeService implementations and the existing holder are valid dispatch locations. + assertTrue(violations(SwitchService.class, TypeServices.class).isEmpty()); + } + + @Test + public void doesNotExemptSimilarlyNamedBusinessClass() { + // A class name containing TypeService is not sufficient to bypass the rule. + assertFalse(violations(NotATypeService.class).isEmpty()); + } + + private static List violations(Class... classes) { + EvaluationResult result = + TSDataTypeSwitchRule.RULE.evaluate(new ClassFileImporter().importClasses(classes)); + return result.getFailureReport().getDetails(); + } + + @Test + public void onlyExemptsServiceLambdaInsideHolder() { + List violations = violations(MixedHolder.class); + assertEquals(2, violations.size()); + assertTrue(violations.stream().anyMatch(message -> message.contains(".business("))); + } + + static class Switches { + int statement(TSDataType type) { + switch (type) { + case INT32: + return 1; + default: + return 0; + } + } + + int expression(TSDataType type) { + return switch (type) { + case DOUBLE -> 2; + default -> 0; + }; + } + } + + static class LambdaSwitch { + ToIntFunction value() { + return type -> + switch (type) { + case FLOAT -> 3; + default -> 0; + }; + } + } + + static class AnonymousSwitch { + static final ToIntFunction VALUE = + new ToIntFunction() { + @Override + public int applyAsInt(TSDataType type) { + return switch (type) { + case INT64 -> 4; + default -> 0; + }; + } + }; + } + + static class OrdinaryUsage { + int value(TSDataType type, Thread.State state) { + int ordinal = type.ordinal(); + int stateValue = + switch (state) { + case NEW -> 1; + default -> 0; + }; + return ordinal + stateValue + TSDataType.INT32.ordinal(); + } + } + + static class SwitchService implements TypeService { + @Override + public Integer call(Type type) { + return dispatch(TSDataType.INT32); + } + + int dispatch(TSDataType type) { + return switch (type) { + case INT32 -> 1; + default -> 0; + }; + } + } + + static class NotATypeService { + int dispatch(TSDataType type) { + return switch (type) { + case INT32 -> 1; + default -> 0; + }; + } + } + + static class MixedHolder { + static final TypeService SERVICE = + type -> + switch (TSDataType.INT32) { + case INT32 -> 1; + default -> 0; + }; + + static final ToIntFunction ORDINARY = + type -> + switch (type) { + case INT32 -> 1; + default -> 0; + }; + + int business(TSDataType type) { + return switch (type) { + case INT32 -> 1; + default -> 0; + }; + } + } +} diff --git a/example/session/src/main/java/org/apache/iotdb/DataMigrationExample.java b/example/session/src/main/java/org/apache/iotdb/DataMigrationExample.java index fb130cce40a6..8df9d9dc2554 100644 --- a/example/session/src/main/java/org/apache/iotdb/DataMigrationExample.java +++ b/example/session/src/main/java/org/apache/iotdb/DataMigrationExample.java @@ -19,6 +19,7 @@ package org.apache.iotdb; +import org.apache.iotdb.commons.i18n.QueryMessages; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.isession.SessionDataSet.DataIterator; @@ -28,6 +29,9 @@ import org.apache.iotdb.session.pool.SessionPool; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -60,6 +64,21 @@ public class DataMigrationExample { // concurrent thread of loading timeseries data private static final int CONCURRENCY = 5; + private static final TypeService COLUMN_VALUE_GETTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> DataIterator::getBoolean; + case INT32 -> DataIterator::getInt; + case INT64, TIMESTAMP -> DataIterator::getLong; + case FLOAT -> DataIterator::getFloat; + case DOUBLE -> DataIterator::getDouble; + case TEXT, STRING -> DataIterator::getString; + case DATE, BLOB, OBJECT -> DataIterator::getObject; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + String.format(QueryMessages.UNSUPPORTED_DATA_TYPE, type.getTypeEnum())); + }; + public static void main(String[] args) throws IoTDBConnectionException, StatementExecutionException, @@ -148,41 +167,11 @@ public Void call() { tablet.addValue(schemaList.get(j).getMeasurementName(), row, null); continue; } - switch (schemaList.get(j).getType()) { - case BOOLEAN: - tablet.addValue( - schemaList.get(j).getMeasurementName(), row, dataIter.getBoolean(j + 2)); - break; - case INT32: - tablet.addValue( - schemaList.get(j).getMeasurementName(), row, dataIter.getInt(j + 2)); - break; - case INT64: - case TIMESTAMP: - tablet.addValue( - schemaList.get(j).getMeasurementName(), row, dataIter.getLong(j + 2)); - break; - case FLOAT: - tablet.addValue( - schemaList.get(j).getMeasurementName(), row, dataIter.getFloat(j + 2)); - break; - case DOUBLE: - tablet.addValue( - schemaList.get(j).getMeasurementName(), row, dataIter.getDouble(j + 2)); - break; - case TEXT: - case STRING: - tablet.addValue( - schemaList.get(j).getMeasurementName(), row, dataIter.getString(j + 2)); - break; - case DATE: - case BLOB: - tablet.addValue( - schemaList.get(j).getMeasurementName(), row, dataIter.getObject(j + 2)); - break; - default: - LOGGER.info("Migration of this type of data is not supported"); - } + Object value = + COLUMN_VALUE_GETTER_SERVICE + .call(Type.fromTsDataType(schemaList.get(j).getType())) + .get(dataIter, j + 2); + tablet.addValue(schemaList.get(j).getMeasurementName(), row, value); } if (tablet.getRowSize() == tablet.getMaxRowNumber()) { writerPool.insertTablet(tablet, true); @@ -210,4 +199,9 @@ public Void call() { return null; } } + + @FunctionalInterface + private interface ColumnValueGetter { + Object get(DataIterator dataIterator, int index) throws StatementExecutionException; + } } diff --git a/example/session/src/main/java/org/apache/iotdb/TabletExample.java b/example/session/src/main/java/org/apache/iotdb/TabletExample.java index f85dba2215b7..79c05ff37b57 100644 --- a/example/session/src/main/java/org/apache/iotdb/TabletExample.java +++ b/example/session/src/main/java/org/apache/iotdb/TabletExample.java @@ -22,6 +22,8 @@ import org.apache.iotdb.session.Session; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -40,6 +42,25 @@ public class TabletExample { private static final String TIME_STR = "time"; + private static final TypeService CSV_VALUE_PARSER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> Boolean::parseBoolean; + case DATE, INT32 -> Integer::parseInt; + case TIMESTAMP, INT64 -> Long::parseLong; + case FLOAT -> Float::parseFloat; + case DOUBLE -> Double::parseDouble; + case STRING, BLOB, TEXT -> BytesUtils::valueOf; + case VECTOR -> + value -> { + throw new IOException( + String.format("data type %s is not yet.", TSDataType.VECTOR)); + }; + default -> + value -> { + throw new IOException("no type"); + }; + }; /** * load csv data. @@ -86,34 +107,8 @@ private static Map> loadCSVData( String measurement = entry.getKey(); TSDataType dataType = entry.getValue(); int idx = columnToIdMap.get(measurement); - switch (dataType) { - case BOOLEAN: - ret.get(measurement).add(Boolean.parseBoolean(items[idx])); - break; - case DATE: - case INT32: - ret.get(measurement).add(Integer.parseInt(items[idx])); - break; - case TIMESTAMP: - case INT64: - ret.get(measurement).add(Long.parseLong(items[idx])); - break; - case FLOAT: - ret.get(measurement).add(Float.parseFloat(items[idx])); - break; - case DOUBLE: - ret.get(measurement).add(Double.parseDouble(items[idx])); - break; - case STRING: - case BLOB: - case TEXT: - ret.get(measurement).add(BytesUtils.valueOf(items[idx])); - break; - case VECTOR: - throw new IOException(String.format("data type %s is not yet.", TSDataType.VECTOR)); - default: - throw new IOException("no type"); - } + ret.get(measurement) + .add(CSV_VALUE_PARSER_SERVICE.call(Type.fromTsDataType(dataType)).parse(items[idx])); } } return ret; @@ -122,6 +117,11 @@ private static Map> loadCSVData( } } + @FunctionalInterface + private interface CsvValueParser { + Object parse(String value) throws IOException; + } + /** * Read csv file and insert tablet to IoTDB * diff --git a/example/udf/pom.xml b/example/udf/pom.xml index bebd303600d1..9c61d350acf5 100644 --- a/example/udf/pom.xml +++ b/example/udf/pom.xml @@ -39,6 +39,12 @@ ${project.version} provided + + org.apache.tsfile + tsfile + ${tsfile.version} + provided + diff --git a/example/udf/src/main/java/org/apache/iotdb/udf/UDAFExample.java b/example/udf/src/main/java/org/apache/iotdb/udf/UDAFExample.java index 540f6bcef5cc..bdf880bed103 100644 --- a/example/udf/src/main/java/org/apache/iotdb/udf/UDAFExample.java +++ b/example/udf/src/main/java/org/apache/iotdb/udf/UDAFExample.java @@ -29,6 +29,8 @@ import org.apache.iotdb.udf.api.utils.ResultValue; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -71,7 +73,7 @@ public void deserialize(byte[] bytes) { } } - private Type dataType; + private ColumnNumericReader valueReader; @Override public void validate(UDFParameterValidator validator) throws UDFException { @@ -82,7 +84,10 @@ public void validate(UDFParameterValidator validator) throws UDFException { @Override public void beforeStart(UDFParameters parameters, UDAFConfigurations configurations) { - dataType = parameters.getDataType(0); + valueReader = + COLUMN_NUMERIC_READER_SERVICE.call( + org.apache.tsfile.read.common.type.Type.fromTsDataType( + TSDataType.getTsDataType(parameters.getDataType(0).getType()))); configurations.setOutputDataType(Type.DOUBLE); } @@ -95,28 +100,15 @@ public State createState() { public void addInput(State state, Column[] columns, BitMap bitMap) { AvgState avgState = (AvgState) state; - switch (dataType) { - case INT32: - addIntInput(avgState, columns, bitMap); - return; - case INT64: - addLongInput(avgState, columns, bitMap); - return; - case FLOAT: - addFloatInput(avgState, columns, bitMap); - return; - case DOUBLE: - addDoubleInput(avgState, columns, bitMap); - return; - case TEXT: - case BOOLEAN: - case TIMESTAMP: - case STRING: - case BLOB: - case DATE: - default: - throw new UnSupportedDataTypeException( - String.format("Unsupported data type in aggregation AVG : %s", dataType)); + int count = columns[0].getPositionCount(); + for (int i = 0; i < count; i++) { + if (bitMap != null && !bitMap.isMarked(i)) { + continue; + } + if (!columns[0].isNull(i)) { + avgState.count++; + avgState.sum += valueReader.read(columns[0], i); + } } } @@ -149,55 +141,21 @@ public void removeState(State state, State removed) { avgState.sum -= avgRhs.sum; } - private void addIntInput(AvgState state, Column[] columns, BitMap bitMap) { - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!columns[0].isNull(i)) { - state.count++; - state.sum += columns[0].getInt(i); - } - } - } - - private void addLongInput(AvgState avgState, Column[] columns, BitMap bitMap) { - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!columns[0].isNull(i)) { - avgState.count++; - avgState.sum += columns[0].getLong(i); - } - } - } - - private void addFloatInput(AvgState avgState, Column[] columns, BitMap bitMap) { - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!columns[0].isNull(i)) { - avgState.count++; - avgState.sum += columns[0].getFloat(i); - } - } - } - - private void addDoubleInput(AvgState avgState, Column[] columns, BitMap bitMap) { - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!columns[0].isNull(i)) { - avgState.count++; - avgState.sum += columns[0].getDouble(i); - } - } + private static final TypeService COLUMN_NUMERIC_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> Column::getInt; + case INT64 -> Column::getLong; + case FLOAT -> Column::getFloat; + case DOUBLE -> Column::getDouble; + default -> + (column, index) -> { + throw new UnSupportedDataTypeException("Unsupported data type: " + type); + }; + }; + + @FunctionalInterface + private interface ColumnNumericReader { + double read(Column column, int index); } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/StatementConstructionHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/StatementConstructionHandler.java index 5836fbf0bacd..b433b48de94b 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/StatementConstructionHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/StatementConstructionHandler.java @@ -23,15 +23,14 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.exception.WriteProcessRejectException; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; -import org.apache.iotdb.rest.i18n.RestMessages; import org.apache.iotdb.rest.protocol.table.v1.model.InsertTabletRequest; +import org.apache.iotdb.rest.protocol.utils.TypeServices; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Locale; @@ -65,109 +64,11 @@ public static InsertTabletStatement constructInsertTabletStatement( for (int columnIndex = 0; columnIndex < columnSize; columnIndex++) { bitMaps[columnIndex] = new BitMap(rowSize); - switch (dataTypes[columnIndex]) { - case BOOLEAN: - boolean[] booleanValues = new boolean[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - Object data = rawData.get(rowIndex).get(columnIndex); - if (data == null) { - bitMaps[columnIndex].mark(rowIndex); - } else { - if ("1".equals(data.toString())) { - booleanValues[rowIndex] = true; - } else if ("0".equals(data.toString())) { - booleanValues[rowIndex] = false; - } else { - booleanValues[rowIndex] = (Boolean) data; - } - } - } - columns[columnIndex] = booleanValues; - break; - case INT32: - case DATE: - int[] intValues = new int[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - Object object = rawData.get(rowIndex).get(columnIndex); - if (object == null) { - bitMaps[columnIndex].mark(rowIndex); - } else if (object instanceof Integer) { - intValues[rowIndex] = (int) object; - } else { - throw new WriteProcessRejectException( - RestMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_0521CEDE - + object.getClass().toString()); - } - } - columns[columnIndex] = intValues; - break; - case INT64: - case TIMESTAMP: - long[] longValues = new long[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - Object object = rawData.get(rowIndex).get(columnIndex); - if (object == null) { - bitMaps[columnIndex].mark(rowIndex); - } else if (object instanceof Integer) { - longValues[rowIndex] = (int) object; - } else if (object instanceof Long) { - longValues[rowIndex] = (long) object; - } else { - throw new WriteProcessRejectException( - RestMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_0521CEDE - + object.getClass().toString()); - } - } - columns[columnIndex] = longValues; - break; - case FLOAT: - float[] floatValues = new float[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - Object data = rawData.get(rowIndex).get(columnIndex); - if (data == null) { - bitMaps[columnIndex].mark(rowIndex); - } else { - floatValues[rowIndex] = Float.parseFloat(String.valueOf(data)); - } - } - columns[columnIndex] = floatValues; - break; - case DOUBLE: - double[] doubleValues = new double[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - if (rawData.get(rowIndex).get(columnIndex) == null) { - bitMaps[columnIndex].mark(rowIndex); - } else { - doubleValues[rowIndex] = - Double.parseDouble(String.valueOf(rawData.get(rowIndex).get(columnIndex))); - } - } - columns[columnIndex] = doubleValues; - break; - case TEXT: - case BLOB: - case STRING: - Binary[] binaryValues = new Binary[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - if (rawData.get(rowIndex).get(columnIndex) == null) { - bitMaps[columnIndex].mark(rowIndex); - binaryValues[rowIndex] = new Binary("".getBytes(StandardCharsets.UTF_8)); - } else { - binaryValues[rowIndex] = - new Binary( - rawData - .get(rowIndex) - .get(columnIndex) - .toString() - .getBytes(StandardCharsets.UTF_8)); - } - } - columns[columnIndex] = binaryValues; - break; - default: - throw new IllegalArgumentException( - RestMessages.INVALID_INPUT + rawDataType.get(columnIndex)); - } + final int column = columnIndex; + columns[columnIndex] = + TypeServices.INSERT_TABLET_COLUMN_WRITER_SERVICE + .call(Type.fromTsDataType(dataTypes[columnIndex])) + .write(rowIndex -> rawData.get(rowIndex).get(column), rowSize, bitMaps[columnIndex]); } insertStatement.setColumns(columns); insertStatement.setBitMaps(bitMaps); diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/utils/InsertRowDataUtils.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/utils/InsertRowDataUtils.java index 08deec903c95..b3b94b8c6ef7 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/utils/InsertRowDataUtils.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/utils/InsertRowDataUtils.java @@ -21,6 +21,8 @@ import org.apache.iotdb.rpc.NoValidValueException; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import java.nio.charset.StandardCharsets; @@ -31,6 +33,58 @@ public class InsertRowDataUtils { + private static final TypeService VALUE_CONVERTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (value, index, mismatchedInfo) -> { + if (!(value instanceof Boolean)) { + mismatchedInfo.put(index, value); + } + return value; + }; + case INT32, DATE -> + (value, index, mismatchedInfo) -> { + if (value instanceof Number number) { + return number.intValue(); + } + mismatchedInfo.put(index, value); + return value; + }; + case INT64, TIMESTAMP -> + (value, index, mismatchedInfo) -> { + if (value instanceof Number number) { + return number.longValue(); + } + mismatchedInfo.put(index, value); + return value; + }; + case FLOAT -> + (value, index, mismatchedInfo) -> { + if (value instanceof Number number) { + return number.floatValue(); + } + mismatchedInfo.put(index, value); + return value; + }; + case DOUBLE -> + (value, index, mismatchedInfo) -> { + if (value instanceof Number number) { + return number.doubleValue(); + } + mismatchedInfo.put(index, value); + return value; + }; + case TEXT, BLOB, STRING -> + (value, index, mismatchedInfo) -> + new Binary(value.toString().getBytes(StandardCharsets.UTF_8)); + case OBJECT, ROW, UNKNOWN, VECTOR -> + (value, index, mismatchedInfo) -> { + throw new IoTDBConnectionException( + MSG_UNSUPPORTED_DATA_TYPE + type.getTypeEnum()); + }; + }; + private static final String ALL_INSERT_DATA_IS_NULL = "All inserted data is null."; public static void filterNullValueAndMeasurement( @@ -77,52 +131,19 @@ public static List reGenValues( continue; } Object val = values.get(i); - switch (types.get(i)) { - case BOOLEAN: - if (!(val instanceof Boolean)) { - mismatchedInfo.put(i, val); - } - break; - case INT32: - case DATE: - if (val instanceof Number) { - values.set(i, ((Number) val).intValue()); - } else { - mismatchedInfo.put(i, val); - } - break; - case INT64: - case TIMESTAMP: - if (val instanceof Number) { - values.set(i, ((Number) val).longValue()); - } else { - mismatchedInfo.put(i, val); - } - break; - case FLOAT: - if (val instanceof Number) { - values.set(i, ((Number) val).floatValue()); - } else { - mismatchedInfo.put(i, val); - } - break; - case DOUBLE: - if (val instanceof Number) { - values.set(i, ((Number) val).doubleValue()); - } else { - mismatchedInfo.put(i, val); - } - break; - case TEXT: - case BLOB: - case STRING: - values.set(i, new Binary(val.toString().getBytes(StandardCharsets.UTF_8))); - break; - default: - throw new IoTDBConnectionException(MSG_UNSUPPORTED_DATA_TYPE + types.get(i)); - } + values.set( + i, + VALUE_CONVERTER_SERVICE + .call(Type.fromTsDataType(types.get(i))) + .convert(val, i, mismatchedInfo)); } return values; } + + @FunctionalInterface + private interface ValueConverter { + Object convert(Object value, int index, Map mismatchedInfo) + throws IoTDBConnectionException; + } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/utils/TypeServices.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/utils/TypeServices.java new file mode 100644 index 000000000000..e1974882471e --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/utils/TypeServices.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.utils; + +import org.apache.iotdb.db.exception.WriteProcessRejectException; +import org.apache.iotdb.rest.i18n.RestMessages; + +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; + +import java.nio.charset.StandardCharsets; + +/** Type-specific column writers used by the REST insert handlers. */ +public final class TypeServices { + + public static final TypeService INSERT_TABLET_COLUMN_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> TypeServices::writeBooleanColumn; + case INT32, DATE -> TypeServices::writeIntColumn; + case INT64, TIMESTAMP -> TypeServices::writeLongColumn; + case FLOAT -> TypeServices::writeFloatColumn; + case DOUBLE -> TypeServices::writeDoubleColumn; + case TEXT, BLOB, STRING -> TypeServices::writeBinaryColumn; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (valueAccessor, rowCount, bitMap) -> { + throw new IllegalArgumentException( + RestMessages.INVALID_INPUT + type.getTypeEnum()); + }; + }; + + static { + INSERT_TABLET_COLUMN_WRITER_SERVICE.check(); + } + + private static Object writeBooleanColumn( + ValueAccessor valueAccessor, int rowCount, BitMap bitMap) { + boolean[] values = new boolean[rowCount]; + for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { + Object data = valueAccessor.get(rowIndex); + if (data == null) { + bitMap.mark(rowIndex); + } else if ("1".equals(data.toString())) { + values[rowIndex] = true; + } else if ("0".equals(data.toString())) { + values[rowIndex] = false; + } else { + values[rowIndex] = (Boolean) data; + } + } + return values; + } + + private static Object writeIntColumn(ValueAccessor valueAccessor, int rowCount, BitMap bitMap) + throws WriteProcessRejectException { + int[] values = new int[rowCount]; + for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { + Object data = valueAccessor.get(rowIndex); + if (data == null) { + bitMap.mark(rowIndex); + } else if (data instanceof Integer) { + values[rowIndex] = (int) data; + } else { + throw unsupportedValue(data); + } + } + return values; + } + + private static Object writeLongColumn(ValueAccessor valueAccessor, int rowCount, BitMap bitMap) + throws WriteProcessRejectException { + long[] values = new long[rowCount]; + for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { + Object data = valueAccessor.get(rowIndex); + if (data == null) { + bitMap.mark(rowIndex); + } else if (data instanceof Integer) { + values[rowIndex] = (int) data; + } else if (data instanceof Long) { + values[rowIndex] = (long) data; + } else { + throw unsupportedValue(data); + } + } + return values; + } + + private static Object writeFloatColumn(ValueAccessor valueAccessor, int rowCount, BitMap bitMap) { + float[] values = new float[rowCount]; + for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { + Object data = valueAccessor.get(rowIndex); + if (data == null) { + bitMap.mark(rowIndex); + } else { + values[rowIndex] = Float.parseFloat(String.valueOf(data)); + } + } + return values; + } + + private static Object writeDoubleColumn( + ValueAccessor valueAccessor, int rowCount, BitMap bitMap) { + double[] values = new double[rowCount]; + for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { + Object data = valueAccessor.get(rowIndex); + if (data == null) { + bitMap.mark(rowIndex); + } else { + values[rowIndex] = Double.parseDouble(String.valueOf(data)); + } + } + return values; + } + + private static Object writeBinaryColumn( + ValueAccessor valueAccessor, int rowCount, BitMap bitMap) { + Binary[] values = new Binary[rowCount]; + for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { + Object data = valueAccessor.get(rowIndex); + if (data == null) { + bitMap.mark(rowIndex); + values[rowIndex] = new Binary(new byte[0]); + } else { + values[rowIndex] = new Binary(data.toString().getBytes(StandardCharsets.UTF_8)); + } + } + return values; + } + + private static WriteProcessRejectException unsupportedValue(Object value) { + return new WriteProcessRejectException("unsupported data type: " + value.getClass()); + } + + @FunctionalInterface + public interface ValueAccessor { + Object get(int rowIndex); + } + + @FunctionalInterface + public interface ColumnWriter { + Object write(ValueAccessor valueAccessor, int rowCount, BitMap bitMap) + throws WriteProcessRejectException; + } + + private TypeServices() {} +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/StatementConstructionHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/StatementConstructionHandler.java index 6f9eee2e87a4..6343d1a8f5aa 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/StatementConstructionHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/StatementConstructionHandler.java @@ -23,14 +23,13 @@ import org.apache.iotdb.db.exception.WriteProcessRejectException; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; -import org.apache.iotdb.rest.i18n.RestMessages; +import org.apache.iotdb.rest.protocol.utils.TypeServices; import org.apache.iotdb.rest.protocol.v1.model.InsertTabletRequest; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Locale; @@ -66,109 +65,11 @@ public static InsertTabletStatement constructInsertTabletStatement( for (int columnIndex = 0; columnIndex < columnSize; columnIndex++) { bitMaps[columnIndex] = new BitMap(rowSize); - switch (dataTypes[columnIndex]) { - case BOOLEAN: - boolean[] booleanValues = new boolean[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - Object data = rawData.get(columnIndex).get(rowIndex); - if (data == null) { - bitMaps[columnIndex].mark(rowIndex); - } else { - if ("1".equals(data.toString())) { - booleanValues[rowIndex] = true; - } else if ("0".equals(data.toString())) { - booleanValues[rowIndex] = false; - } else { - booleanValues[rowIndex] = (Boolean) data; - } - } - } - columns[columnIndex] = booleanValues; - break; - case INT32: - case DATE: - int[] intValues = new int[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - Object object = rawData.get(columnIndex).get(rowIndex); - if (object == null) { - bitMaps[columnIndex].mark(rowIndex); - } else if (object instanceof Integer) { - intValues[rowIndex] = (int) object; - } else { - throw new WriteProcessRejectException( - RestMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_0521CEDE - + object.getClass().toString()); - } - } - columns[columnIndex] = intValues; - break; - case INT64: - case TIMESTAMP: - long[] longValues = new long[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - Object object = rawData.get(columnIndex).get(rowIndex); - if (object == null) { - bitMaps[columnIndex].mark(rowIndex); - } else if (object instanceof Integer) { - longValues[rowIndex] = (int) object; - } else if (object instanceof Long) { - longValues[rowIndex] = (long) object; - } else { - throw new WriteProcessRejectException( - RestMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_0521CEDE - + object.getClass().toString()); - } - } - columns[columnIndex] = longValues; - break; - case FLOAT: - float[] floatValues = new float[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - Object data = rawData.get(columnIndex).get(rowIndex); - if (data == null) { - bitMaps[columnIndex].mark(rowIndex); - } else { - floatValues[rowIndex] = Float.parseFloat(String.valueOf(data)); - } - } - columns[columnIndex] = floatValues; - break; - case DOUBLE: - double[] doubleValues = new double[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - if (rawData.get(columnIndex).get(rowIndex) == null) { - bitMaps[columnIndex].mark(rowIndex); - } else { - doubleValues[rowIndex] = - Double.parseDouble(String.valueOf(rawData.get(columnIndex).get(rowIndex))); - } - } - columns[columnIndex] = doubleValues; - break; - case TEXT: - case BLOB: - case STRING: - Binary[] binaryValues = new Binary[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - if (rawData.get(columnIndex).get(rowIndex) == null) { - bitMaps[columnIndex].mark(rowIndex); - binaryValues[rowIndex] = new Binary("".getBytes(StandardCharsets.UTF_8)); - } else { - binaryValues[rowIndex] = - new Binary( - rawData - .get(columnIndex) - .get(rowIndex) - .toString() - .getBytes(StandardCharsets.UTF_8)); - } - } - columns[columnIndex] = binaryValues; - break; - default: - throw new IllegalArgumentException( - RestMessages.INVALID_INPUT + rawDataType.get(columnIndex)); - } + final int column = columnIndex; + columns[columnIndex] = + TypeServices.INSERT_TABLET_COLUMN_WRITER_SERVICE + .call(Type.fromTsDataType(dataTypes[columnIndex])) + .write(rowIndex -> rawData.get(column).get(rowIndex), rowSize, bitMaps[columnIndex]); } insertStatement.setTimes( diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/StatementConstructionHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/StatementConstructionHandler.java index 50d365535ae2..770648a8cdb5 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/StatementConstructionHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/StatementConstructionHandler.java @@ -26,17 +26,16 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; -import org.apache.iotdb.rest.i18n.RestMessages; import org.apache.iotdb.rest.protocol.utils.InsertRowDataUtils; +import org.apache.iotdb.rest.protocol.utils.TypeServices; import org.apache.iotdb.rest.protocol.v2.model.InsertRecordsRequest; import org.apache.iotdb.rest.protocol.v2.model.InsertTabletRequest; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -76,109 +75,11 @@ public static InsertTabletStatement constructInsertTabletStatement( for (int columnIndex = 0; columnIndex < columnSize; columnIndex++) { bitMaps[columnIndex] = new BitMap(rowSize); - switch (dataTypes[columnIndex]) { - case BOOLEAN: - boolean[] booleanValues = new boolean[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - Object data = rawData.get(columnIndex).get(rowIndex); - if (data == null) { - bitMaps[columnIndex].mark(rowIndex); - } else { - if ("1".equals(data.toString())) { - booleanValues[rowIndex] = true; - } else if ("0".equals(data.toString())) { - booleanValues[rowIndex] = false; - } else { - booleanValues[rowIndex] = (Boolean) data; - } - } - } - columns[columnIndex] = booleanValues; - break; - case INT32: - case DATE: - int[] intValues = new int[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - Object object = rawData.get(columnIndex).get(rowIndex); - if (object == null) { - bitMaps[columnIndex].mark(rowIndex); - } else if (object instanceof Integer) { - intValues[rowIndex] = (int) object; - } else { - throw new WriteProcessRejectException( - RestMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_0521CEDE - + object.getClass().toString()); - } - } - columns[columnIndex] = intValues; - break; - case INT64: - case TIMESTAMP: - long[] longValues = new long[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - Object object = rawData.get(columnIndex).get(rowIndex); - if (object == null) { - bitMaps[columnIndex].mark(rowIndex); - } else if (object instanceof Integer) { - longValues[rowIndex] = (int) object; - } else if (object instanceof Long) { - longValues[rowIndex] = (long) object; - } else { - throw new WriteProcessRejectException( - RestMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_0521CEDE - + object.getClass().toString()); - } - } - columns[columnIndex] = longValues; - break; - case FLOAT: - float[] floatValues = new float[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - Object data = rawData.get(columnIndex).get(rowIndex); - if (data == null) { - bitMaps[columnIndex].mark(rowIndex); - } else { - floatValues[rowIndex] = Float.parseFloat(String.valueOf(data)); - } - } - columns[columnIndex] = floatValues; - break; - case DOUBLE: - double[] doubleValues = new double[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - if (rawData.get(columnIndex).get(rowIndex) == null) { - bitMaps[columnIndex].mark(rowIndex); - } else { - doubleValues[rowIndex] = - Double.parseDouble(String.valueOf(rawData.get(columnIndex).get(rowIndex))); - } - } - columns[columnIndex] = doubleValues; - break; - case TEXT: - case BLOB: - case STRING: - Binary[] binaryValues = new Binary[rowSize]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - if (rawData.get(columnIndex).get(rowIndex) == null) { - bitMaps[columnIndex].mark(rowIndex); - binaryValues[rowIndex] = new Binary("".getBytes(StandardCharsets.UTF_8)); - } else { - binaryValues[rowIndex] = - new Binary( - rawData - .get(columnIndex) - .get(rowIndex) - .toString() - .getBytes(StandardCharsets.UTF_8)); - } - } - columns[columnIndex] = binaryValues; - break; - default: - throw new IllegalArgumentException( - RestMessages.INVALID_INPUT + rawDataType.get(columnIndex)); - } + final int column = columnIndex; + columns[columnIndex] = + TypeServices.INSERT_TABLET_COLUMN_WRITER_SERVICE + .call(Type.fromTsDataType(dataTypes[columnIndex])) + .write(rowIndex -> rawData.get(column).get(rowIndex), rowSize, bitMaps[columnIndex]); } insertStatement.setTimes( diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/v2/handler/FastLastHandlerTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/v2/handler/FastLastHandlerTest.java index a275813c3938..751cde923c23 100644 --- a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/v2/handler/FastLastHandlerTest.java +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/v2/handler/FastLastHandlerTest.java @@ -28,8 +28,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.TsPrimitiveType; import org.junit.Test; import jakarta.ws.rs.core.Response; @@ -114,6 +114,7 @@ private static Pair newEntry(String name, long timest } return new Pair<>( TSDataType.INT64, - new TimeValuePair(timestamp, TsPrimitiveType.getByType(TSDataType.INT64, timestamp))); + new TimeValuePair( + timestamp, Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(timestamp))); } } diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/TwoSum.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/TwoSum.java index 6ab1ff6ab20b..e13b22f9ed97 100644 --- a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/TwoSum.java +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/TwoSum.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.query.udf.example; +import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; @@ -27,8 +28,24 @@ import org.apache.iotdb.udf.api.customizer.strategy.MappableRowByRowAccessStrategy; import org.apache.iotdb.udf.api.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; + public class TwoSum implements UDTF { - private Type dataType; + + private static final TypeService TWO_SUM_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> row -> row.getInt(0) + row.getInt(1); + case INT64 -> row -> row.getLong(0) + row.getLong(1); + case FLOAT -> row -> row.getFloat(0) + row.getFloat(1); + case DOUBLE -> row -> row.getDouble(0) + row.getDouble(1); + default -> + row -> { + throw new Exception(); + }; + }; + + private org.apache.tsfile.read.common.type.Type dataType; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -40,25 +57,22 @@ public void validate(UDFParameterValidator validator) throws Exception { @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) { - dataType = parameters.getDataType(0); + final Type udfDataType = parameters.getDataType(0); + dataType = + org.apache.tsfile.read.common.type.Type.fromTsDataType( + UDFDataTypeTransformer.transformToTsDataType(udfDataType)); configurations .setAccessStrategy(new MappableRowByRowAccessStrategy()) - .setOutputDataType(dataType); + .setOutputDataType(udfDataType); } @Override public Object transform(Row row) throws Exception { - switch (dataType) { - case INT32: - return row.getInt(0) + row.getInt(1); - case INT64: - return row.getLong(0) + row.getLong(1); - case FLOAT: - return row.getFloat(0) + row.getFloat(1); - case DOUBLE: - return row.getDouble(0) + row.getDouble(1); - default: - throw new Exception(); - } + return TWO_SUM_TRANSFORMER_SERVICE.call(dataType).transform(row); + } + + @FunctionalInterface + private interface TwoSumTransformer { + Object transform(Row row) throws Exception; } } diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFAvg.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFAvg.java index a8aab5d0a452..010243a77ce4 100644 --- a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFAvg.java +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFAvg.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.query.udf.example; +import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; import org.apache.iotdb.udf.api.State; import org.apache.iotdb.udf.api.UDAF; import org.apache.iotdb.udf.api.customizer.config.UDAFConfigurations; @@ -29,12 +30,30 @@ import org.apache.iotdb.udf.api.utils.ResultValue; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; public class UDAFAvg implements UDAF { + + private static final TypeService COLUMN_VALUE_GETTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> Column::getInt; + case INT64 -> Column::getLong; + case FLOAT -> Column::getFloat; + case DOUBLE -> Column::getDouble; + default -> + (column, index) -> { + throw new UnSupportedDataTypeException( + String.format( + "Unsupported data type in aggregation AVG : %s", type.getTypeEnum())) + .setChecked(false); + }; + }; + static class AvgState implements State { double sum; @@ -63,7 +82,7 @@ public void deserialize(byte[] bytes) { } } - private Type dataType; + private org.apache.tsfile.read.common.type.Type dataType; @Override public void validate(UDFParameterValidator validator) throws UDFException { @@ -74,7 +93,9 @@ public void validate(UDFParameterValidator validator) throws UDFException { @Override public void beforeStart(UDFParameters parameters, UDAFConfigurations configurations) { - dataType = parameters.getDataType(0); + dataType = + org.apache.tsfile.read.common.type.Type.fromTsDataType( + UDFDataTypeTransformer.transformToTsDataType(parameters.getDataType(0))); configurations.setOutputDataType(Type.DOUBLE); } @@ -86,29 +107,17 @@ public State createState() { @Override public void addInput(State state, Column[] columns, BitMap bitMap) { AvgState avgState = (AvgState) state; - - switch (dataType) { - case INT32: - addIntInput(avgState, columns, bitMap); - return; - case INT64: - addLongInput(avgState, columns, bitMap); - return; - case FLOAT: - addFloatInput(avgState, columns, bitMap); - return; - case DOUBLE: - addDoubleInput(avgState, columns, bitMap); - return; - case TEXT: - case STRING: - case BLOB: - case BOOLEAN: - case TIMESTAMP: - case DATE: - default: - throw new UnSupportedDataTypeException( - String.format("Unsupported data type in aggregation AVG : %s", dataType)); + final Column column = columns[0]; + final ColumnValueGetter valueGetter = COLUMN_VALUE_GETTER_SERVICE.call(dataType); + final int count = column.getPositionCount(); + for (int i = 0; i < count; i++) { + if (bitMap != null && !bitMap.isMarked(i)) { + continue; + } + if (!column.isNull(i)) { + avgState.count++; + avgState.sum += valueGetter.getValue(column, i); + } } } @@ -141,55 +150,8 @@ public void removeState(State state, State removed) { avgState.sum -= avgRhs.sum; } - private void addIntInput(AvgState state, Column[] columns, BitMap bitMap) { - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!columns[0].isNull(i)) { - state.count++; - state.sum += columns[0].getInt(i); - } - } - } - - private void addLongInput(AvgState avgState, Column[] columns, BitMap bitMap) { - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!columns[0].isNull(i)) { - avgState.count++; - avgState.sum += columns[0].getLong(i); - } - } - } - - private void addFloatInput(AvgState avgState, Column[] columns, BitMap bitMap) { - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!columns[0].isNull(i)) { - avgState.count++; - avgState.sum += columns[0].getFloat(i); - } - } - } - - private void addDoubleInput(AvgState avgState, Column[] columns, BitMap bitMap) { - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!columns[0].isNull(i)) { - avgState.count++; - avgState.sum += columns[0].getDouble(i); - } - } + @FunctionalInterface + private interface ColumnValueGetter { + double getValue(Column column, int index); } } diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFSum.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFSum.java index a829c9f2c477..797b4df9125a 100644 --- a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFSum.java +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFSum.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.query.udf.example; +import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; import org.apache.iotdb.udf.api.State; import org.apache.iotdb.udf.api.UDAF; import org.apache.iotdb.udf.api.customizer.config.UDAFConfigurations; @@ -29,12 +30,30 @@ import org.apache.iotdb.udf.api.utils.ResultValue; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; public class UDAFSum implements UDAF { + + private static final TypeService COLUMN_VALUE_GETTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> Column::getInt; + case INT64 -> Column::getLong; + case FLOAT -> Column::getFloat; + case DOUBLE -> Column::getDouble; + default -> + (column, index) -> { + throw new UnSupportedDataTypeException( + String.format( + "Unsupported data type in aggregation AVG : %s", type.getTypeEnum())) + .setChecked(false); + }; + }; + static class SumState implements State { double sum = 0; @@ -63,7 +82,7 @@ public void deserialize(byte[] bytes) { } } - private Type dataType; + private org.apache.tsfile.read.common.type.Type dataType; @Override public void validate(UDFParameterValidator validator) throws UDFException { @@ -74,7 +93,9 @@ public void validate(UDFParameterValidator validator) throws UDFException { @Override public void beforeStart(UDFParameters parameters, UDAFConfigurations configurations) { - dataType = parameters.getDataType(0); + dataType = + org.apache.tsfile.read.common.type.Type.fromTsDataType( + UDFDataTypeTransformer.transformToTsDataType(parameters.getDataType(0))); configurations.setOutputDataType(Type.DOUBLE); } @@ -86,29 +107,17 @@ public State createState() { @Override public void addInput(State state, Column[] columns, BitMap bitMap) { SumState sumState = (SumState) state; - - switch (dataType) { - case INT32: - addIntInput(sumState, columns, bitMap); - return; - case INT64: - addLongInput(sumState, columns, bitMap); - return; - case FLOAT: - addFloatInput(sumState, columns, bitMap); - return; - case DOUBLE: - addDoubleInput(sumState, columns, bitMap); - return; - case TEXT: - case STRING: - case BLOB: - case BOOLEAN: - case TIMESTAMP: - case DATE: - default: - throw new UnSupportedDataTypeException( - String.format("Unsupported data type in aggregation AVG : %s", dataType)); + final Column column = columns[0]; + final ColumnValueGetter valueGetter = COLUMN_VALUE_GETTER_SERVICE.call(dataType); + final int count = column.getPositionCount(); + for (int i = 0; i < count; i++) { + if (bitMap != null && !bitMap.isMarked(i)) { + continue; + } + if (!column.isNull(i)) { + sumState.initResult = true; + sumState.sum += valueGetter.getValue(column, i); + } } } @@ -140,55 +149,8 @@ public void removeState(State state, State removed) { sumState.sum -= sumRhs.sum; } - private void addIntInput(SumState state, Column[] columns, BitMap bitMap) { - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!columns[0].isNull(i)) { - state.initResult = true; - state.sum += columns[0].getInt(i); - } - } - } - - private void addLongInput(SumState state, Column[] columns, BitMap bitMap) { - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!columns[0].isNull(i)) { - state.initResult = true; - state.sum += columns[0].getLong(i); - } - } - } - - private void addFloatInput(SumState state, Column[] columns, BitMap bitMap) { - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!columns[0].isNull(i)) { - state.initResult = true; - state.sum += columns[0].getFloat(i); - } - } - } - - private void addDoubleInput(SumState state, Column[] columns, BitMap bitMap) { - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!columns[0].isNull(i)) { - state.initResult = true; - state.sum += columns[0].getDouble(i); - } - } + @FunctionalInterface + private interface ColumnValueGetter { + double getValue(Column column, int index); } } diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/access/Row.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/access/Row.java index e2dc13a39ab2..c1b466786cdc 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/access/Row.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/access/Row.java @@ -68,10 +68,11 @@ public interface Row { float getFloat(int columnIndex) throws IOException; /** - * Returns the double value at the specified column in this row. + * Returns the numeric value at the specified column in this row as a double. * - *

Users need to ensure that the data type of the specified column is {@code - * TSDataType.DOUBLE}. + *

Users need to ensure that the data type of the specified column is {@code TSDataType.INT32}, + * {@code TSDataType.INT64}, {@code TSDataType.FLOAT}, or {@code TSDataType.DOUBLE}, and that the + * value is not null. INT64 values may lose precision when converted to double. * * @param columnIndex index of the specified column * @return the double value at the specified column in this row diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java index a082a1968ce6..4033ff9ac761 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java @@ -73,7 +73,7 @@ public double getDouble(int columnIndex) { if (columnIndex >= size()) { throw new IndexOutOfBoundsException(UdfApiMessages.INDEX_OUT_OF_BOUND); } - return (double) rowRecord[columnIndex]; + return ((Number) rowRecord[columnIndex]).doubleValue(); } @Override @@ -129,30 +129,10 @@ private static Type transformToUDFDataType(TSDataType tsDataType) { if (tsDataType == null) { return null; } - byte type = tsDataType.getType(); - switch (type) { - case 0: - return Type.BOOLEAN; - case 1: - return Type.INT32; - case 2: - return Type.INT64; - case 3: - return Type.FLOAT; - case 4: - return Type.DOUBLE; - case 5: - return Type.TEXT; - case 8: - return Type.TIMESTAMP; - case 9: - return Type.DATE; - case 10: - return Type.BLOB; - case 11: - return Type.STRING; - default: - throw new IllegalArgumentException(UdfApiMessages.INVALID_INPUT + type); + try { + return Type.valueOf(tsDataType.getType()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(UdfApiMessages.INVALID_INPUT + tsDataType.getType(), e); } } } diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java index 0329812f2d4d..0b0f30d970d9 100644 --- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java @@ -50,6 +50,10 @@ public final class CliMessages { public static final String FAILED_TO_WRITE_DATA = "Failed to write data to file: {}"; public static final String FAILED_TO_CREATE_FILE = "Failed to create file: {}"; + // AbstractCli + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_B411C29E = + "Unsupported data type: %s"; + // AbstractDataTool public static final String USE_HELP_FOR_MORE = "Use -help for more information"; diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java index 61a14d94d588..a260732ab554 100644 --- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java @@ -50,6 +50,10 @@ public final class CliMessages { public static final String FAILED_TO_WRITE_DATA = "向文件写入数据失败：{}"; public static final String FAILED_TO_CREATE_FILE = "创建文件失败：{}"; + // AbstractCli + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_B411C29E = + "不支持的数据类型：%s"; + // AbstractDataTool public static final String USE_HELP_FOR_MORE = "使用 -help 获取更多信息"; diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java index 088131c25702..aad1074860fb 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java @@ -35,8 +35,11 @@ import org.apache.commons.cli.Options; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.lang3.ArrayUtils; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.BufferedReader; import java.io.InputStreamReader; @@ -56,6 +59,38 @@ public abstract class AbstractCli { + static String timestampPrecision = "ms"; + static String timeFormat = RpcUtils.DEFAULT_TIME_FORMAT; + + private static final TypeService SQL_VALUE_GETTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, STRING, OBJECT -> + (resultSet, columnIndex, zoneId) -> resultSet.getString(columnIndex); + case BLOB -> + (resultSet, columnIndex, zoneId) -> { + byte[] value = resultSet.getBytes(columnIndex); + return value == null ? null : BytesUtils.parseBlobByteArrayToString(value); + }; + case DATE -> + (resultSet, columnIndex, zoneId) -> { + int value = resultSet.getInt(columnIndex); + return resultSet.wasNull() ? null : DateUtils.formatDate(value); + }; + case TIMESTAMP -> + (resultSet, columnIndex, zoneId) -> { + long value = resultSet.getLong(columnIndex); + return resultSet.wasNull() + ? null + : RpcUtils.formatDatetime(timeFormat, timestampPrecision, value, zoneId); + }; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + String.format( + CliMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_B411C29E, + type.getTypeEnum())); + }; + static final String HOST_ARGS = "h"; static final String HOST_NAME = "host"; @@ -119,8 +154,6 @@ public abstract class AbstractCli { static int maxPrintRowCount = 1000; private static int fetchSize = 1000; static int queryTimeout = 0; - static String timestampPrecision = "ms"; - static String timeFormat = RpcUtils.DEFAULT_TIME_FORMAT; private static boolean continuePrint = false; private static int lineCount = 0; @@ -794,40 +827,14 @@ private static List> cacheResult( private static String getStringByColumnIndex( IoTDBJDBCResultSet resultSet, int columnIndex, ZoneId zoneId) throws SQLException { TSDataType type = resultSet.getColumnTypeByIndex(columnIndex); - switch (type) { - case BOOLEAN: - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - case TEXT: - case STRING: - case OBJECT: - return resultSet.getString(columnIndex); - case BLOB: - byte[] v = resultSet.getBytes(columnIndex); - if (v == null) { - return null; - } else { - return BytesUtils.parseBlobByteArrayToString(v); - } - case DATE: - int intValue = resultSet.getInt(columnIndex); - if (resultSet.wasNull()) { - return null; - } else { - return DateUtils.formatDate(intValue); - } - case TIMESTAMP: - long longValue = resultSet.getLong(columnIndex); - if (resultSet.wasNull()) { - return null; - } else { - return RpcUtils.formatDatetime(timeFormat, timestampPrecision, longValue, zoneId); - } - default: - return null; - } + return SQL_VALUE_GETTER_SERVICE + .call(Type.fromTsDataType(type)) + .get(resultSet, columnIndex, zoneId); + } + + @FunctionalInterface + private interface SqlValueGetter { + String get(ResultSet resultSet, int columnIndex, ZoneId zoneId) throws SQLException; } private static List> cacheTracingInfo(ResultSet resultSet, List maxSizeList) diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java index b928fbd98674..06d02a7de834 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java @@ -56,7 +56,10 @@ import org.apache.tsfile.external.commons.lang3.StringUtils; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.UnSupportedDataTypeException; import org.jline.reader.LineReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,6 +91,35 @@ public abstract class AbstractDataTool { + private static final TypeService VALUE_PARSER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case TEXT, STRING -> + value -> { + if (value.startsWith("\"") && value.endsWith("\"")) { + return value.substring(1, value.length() - 1); + } + return value; + }; + case BOOLEAN -> + value -> + !"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value) + ? null + : Boolean.parseBoolean(value); + case INT32 -> value -> Integer.parseInt(value); + case INT64, TIMESTAMP -> value -> Long.parseLong(value); + case FLOAT -> value -> Float.parseFloat(value); + case DOUBLE -> value -> Double.parseDouble(value); + case DATE -> value -> LocalDate.parse(value); + case BLOB -> + value -> new Binary(parseHexStringToByteArray(value.replaceFirst("0x", ""))); + case OBJECT, ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + String.format( + CliMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_B411C29E, + type.getTypeEnum())); + }; + protected static String host; protected static String port; protected static String table; @@ -467,40 +499,17 @@ private static boolean isConvertFloatPrecisionLack(String s) { */ protected static Object typeTrans(String value, TSDataType type) { try { - switch (type) { - case TEXT: - case STRING: - if (value.startsWith("\"") && value.endsWith("\"")) { - return value.substring(1, value.length() - 1); - } - return value; - case BOOLEAN: - if (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value)) { - return null; - } - return Boolean.parseBoolean(value); - case INT32: - return Integer.parseInt(value); - case INT64: - return Long.parseLong(value); - case FLOAT: - return Float.parseFloat(value); - case DOUBLE: - return Double.parseDouble(value); - case TIMESTAMP: - return Long.parseLong(value); - case DATE: - return LocalDate.parse(value); - case BLOB: - return new Binary(parseHexStringToByteArray(value.replaceFirst("0x", ""))); - default: - return null; - } + return VALUE_PARSER_SERVICE.call(Type.fromTsDataType(type)).parse(value); } catch (NumberFormatException e) { return null; } } + @FunctionalInterface + private interface ValueParser { + Object parse(String value); + } + private static byte[] parseHexStringToByteArray(String hexString) { byte[] bytes = new byte[hexString.length() / 2]; for (int i = 0; i < hexString.length(); i += 2) { diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java index a926103222a2..5076afd52d05 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java @@ -30,8 +30,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; import java.sql.Timestamp; @@ -161,40 +161,9 @@ private RowRecord constructRowRecordFromValueArray() throws StatementExecutionEx if (!ioTDBRpcDataSet.isNull(columnIndex)) { TSDataType dataType = ioTDBRpcDataSet.getDataType(columnIndex); field = new Field(dataType); - switch (dataType) { - case BOOLEAN: - boolean booleanValue = ioTDBRpcDataSet.getBoolean(columnIndex); - field.setBoolV(booleanValue); - break; - case INT32: - case DATE: - int intValue = ioTDBRpcDataSet.getInt(columnIndex); - field.setIntV(intValue); - break; - case INT64: - case TIMESTAMP: - long longValue = ioTDBRpcDataSet.getLong(columnIndex); - field.setLongV(longValue); - break; - case FLOAT: - float floatValue = ioTDBRpcDataSet.getFloat(columnIndex); - field.setFloatV(floatValue); - break; - case DOUBLE: - double doubleValue = ioTDBRpcDataSet.getDouble(columnIndex); - field.setDoubleV(doubleValue); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - field.setBinaryV(ioTDBRpcDataSet.getBinary(columnIndex)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - ISessionMessages.EXCEPTION_DATA_TYPE_ARG_NOT_SUPPORTED_31213160, dataType)); - } + TypeServices.FIELD_VALUE_READER_SERVICE + .call(Type.fromTsDataType(dataType)) + .read(ioTDBRpcDataSet, columnIndex, field); } else { field = new Field(null); } diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/TypeServices.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/TypeServices.java new file mode 100644 index 000000000000..09b08977d099 --- /dev/null +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/TypeServices.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.isession; + +import org.apache.iotdb.isession.i18n.ISessionMessages; +import org.apache.iotdb.rpc.IoTDBRpcDataSet; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +/** Type-specific RPC result readers that preserve primitive access without intermediate boxing. */ +final class TypeServices { + + static final TypeService FIELD_VALUE_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (dataSet, columnIndex, field) -> field.setBoolV(dataSet.getBoolean(columnIndex)); + case INT32, DATE -> + (dataSet, columnIndex, field) -> field.setIntV(dataSet.getInt(columnIndex)); + case INT64, TIMESTAMP -> + (dataSet, columnIndex, field) -> field.setLongV(dataSet.getLong(columnIndex)); + case FLOAT -> + (dataSet, columnIndex, field) -> field.setFloatV(dataSet.getFloat(columnIndex)); + case DOUBLE -> + (dataSet, columnIndex, field) -> field.setDoubleV(dataSet.getDouble(columnIndex)); + case TEXT, BLOB, STRING, OBJECT -> + (dataSet, columnIndex, field) -> field.setBinaryV(dataSet.getBinary(columnIndex)); + case ROW, UNKNOWN, VECTOR -> + (dataSet, columnIndex, field) -> { + throw new UnSupportedDataTypeException( + String.format( + ISessionMessages.EXCEPTION_DATA_TYPE_ARG_NOT_SUPPORTED_31213160, + type.getTypeEnum())); + }; + }; + + static { + FIELD_VALUE_READER_SERVICE.check(); + } + + private TypeServices() {} + + @FunctionalInterface + interface FieldValueReader { + + void read(IoTDBRpcDataSet dataSet, int columnIndex, Field field) + throws StatementExecutionException; + } +} diff --git a/iotdb-client/jdbc/src/main/i18n/en/org/apache/iotdb/jdbc/i18n/JdbcMessages.java b/iotdb-client/jdbc/src/main/i18n/en/org/apache/iotdb/jdbc/i18n/JdbcMessages.java index 43e0ef6cb46f..dc26ffc77b58 100644 --- a/iotdb-client/jdbc/src/main/i18n/en/org/apache/iotdb/jdbc/i18n/JdbcMessages.java +++ b/iotdb-client/jdbc/src/main/i18n/en/org/apache/iotdb/jdbc/i18n/JdbcMessages.java @@ -91,6 +91,8 @@ public final class JdbcMessages { // IoTDBAbstractDatabaseMetadata public static final String NO_DATA_TYPE_MATCHED = "No data type was matched: {}"; + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_B411C29E = + "Unsupported data type: %s"; public static final String GET_READ_ONLY_ERROR = "Get is readOnly error: {}"; public static final String CANNOT_GET_READ_ONLY_MODE = "Can not get the read-only mode"; public static final String GET_SYSTEM_FUNCTIONS_ERROR = "Get system functions error: {}"; diff --git a/iotdb-client/jdbc/src/main/i18n/zh/org/apache/iotdb/jdbc/i18n/JdbcMessages.java b/iotdb-client/jdbc/src/main/i18n/zh/org/apache/iotdb/jdbc/i18n/JdbcMessages.java index 1aeededfc900..7d5fb9ec2fe0 100644 --- a/iotdb-client/jdbc/src/main/i18n/zh/org/apache/iotdb/jdbc/i18n/JdbcMessages.java +++ b/iotdb-client/jdbc/src/main/i18n/zh/org/apache/iotdb/jdbc/i18n/JdbcMessages.java @@ -91,6 +91,7 @@ public final class JdbcMessages { // IoTDBAbstractDatabaseMetadata public static final String NO_DATA_TYPE_MATCHED = "没有匹配的数据类型：{}"; + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_B411C29E = "不支持的数据类型：%s"; public static final String GET_READ_ONLY_ERROR = "获取只读模式错误：{}"; public static final String CANNOT_GET_READ_ONLY_MODE = "无法获取只读模式"; public static final String GET_SYSTEM_FUNCTIONS_ERROR = "获取系统函数错误：{}"; diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/GroupedLSBWatermarkEncoder.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/GroupedLSBWatermarkEncoder.java index 58579f7d437d..9f56f34e4e82 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/GroupedLSBWatermarkEncoder.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/GroupedLSBWatermarkEncoder.java @@ -22,9 +22,10 @@ import org.apache.iotdb.jdbc.i18n.JdbcMessages; import org.apache.thrift.EncodingUtils; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import java.math.BigInteger; import java.security.MessageDigest; @@ -32,6 +33,25 @@ import java.util.List; public class GroupedLSBWatermarkEncoder implements WatermarkEncoder { + + private static final TypeService FIELD_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (encoder, field, timestamp) -> + field.setIntV(encoder.encodeInt(field.getIntV(), timestamp)); + case INT64, TIMESTAMP -> + (encoder, field, timestamp) -> + field.setLongV(encoder.encodeLong(field.getLongV(), timestamp)); + case FLOAT -> + (encoder, field, timestamp) -> + field.setFloatV(encoder.encodeFloat(field.getFloatV(), timestamp)); + case DOUBLE -> + (encoder, field, timestamp) -> + field.setDoubleV(encoder.encodeDouble(field.getDoubleV(), timestamp)); + default -> (encoder, field, timestamp) -> {}; + }; + private String secretKey; private String bitString; private int markRate = 2; @@ -116,33 +136,15 @@ public RowRecord encodeRecord(RowRecord rowRecord) { if (field == null || field.getDataType() == null) { continue; } - TSDataType dataType = field.getDataType(); - switch (dataType) { - case INT32: - case DATE: - int originIntValue = field.getIntV(); - field.setIntV(encodeInt(originIntValue, timestamp)); - break; - case INT64: - case TIMESTAMP: - long originLongValue = field.getLongV(); - field.setLongV(encodeLong(originLongValue, timestamp)); - break; - case FLOAT: - float originFloatValue = field.getFloatV(); - field.setFloatV(encodeFloat(originFloatValue, timestamp)); - break; - case DOUBLE: - double originDoubleValue = field.getDoubleV(); - field.setDoubleV(encodeDouble(originDoubleValue, timestamp)); - break; - case BLOB: - case STRING: - case BOOLEAN: - case TEXT: - default: - } + FIELD_ENCODER_SERVICE + .call(Type.fromTsDataType(field.getDataType())) + .encode(this, field, timestamp); } return rowRecord; } + + @FunctionalInterface + private interface FieldEncoder { + void encode(GroupedLSBWatermarkEncoder encoder, Field field, long timestamp); + } } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java index 0f81bca6069a..0c50512e80b0 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java @@ -23,12 +23,16 @@ import org.apache.iotdb.service.rpc.thrift.IClientRPCService; import org.apache.thrift.TException; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.column.TsBlockSerde; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.UnSupportedDataTypeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +56,25 @@ public abstract class IoTDBAbstractDatabaseMetadata implements DatabaseMetaData { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBAbstractDatabaseMetadata.class); + + private static final TypeService COLUMN_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case TEXT, STRING, BLOB, OBJECT -> + (builder, value) -> + builder.writeBinary(new Binary(value.toString(), TSFileConfig.STRING_CHARSET)); + case FLOAT -> (builder, value) -> builder.writeFloat((float) value); + case INT32, DATE -> (builder, value) -> builder.writeInt((int) value); + case INT64, TIMESTAMP -> (builder, value) -> builder.writeLong((long) value); + case DOUBLE -> (builder, value) -> builder.writeDouble((double) value); + case BOOLEAN -> (builder, value) -> builder.writeBoolean((boolean) value); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + String.format( + JdbcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_B411C29E, + type.getTypeEnum())); + }; + private static final String METHOD_NOT_SUPPORTED_STRING = JdbcMessages.METHOD_NOT_SUPPORTED; protected static final String CONVERT_ERROR_MSG = "Convert tsBlock error: {}"; @@ -393,37 +416,9 @@ public static ByteBuffer convertTsBlock( tsBlockBuilder.getTimeColumnBuilder().writeLong(0); for (int j = 0; j < tsDataTypeList.size(); j++) { TSDataType columnType = tsDataTypeList.get(j); - switch (columnType) { - case TEXT: - case STRING: - case BLOB: - case OBJECT: - tsBlockBuilder - .getColumnBuilder(j) - .writeBinary( - new Binary(valuesInRow.get(j).toString(), TSFileConfig.STRING_CHARSET)); - break; - case FLOAT: - tsBlockBuilder.getColumnBuilder(j).writeFloat((float) valuesInRow.get(j)); - break; - case INT32: - case DATE: - tsBlockBuilder.getColumnBuilder(j).writeInt((int) valuesInRow.get(j)); - break; - case INT64: - case TIMESTAMP: - tsBlockBuilder.getColumnBuilder(j).writeLong((long) valuesInRow.get(j)); - break; - case DOUBLE: - tsBlockBuilder.getColumnBuilder(j).writeDouble((double) valuesInRow.get(j)); - break; - case BOOLEAN: - tsBlockBuilder.getColumnBuilder(j).writeBoolean((boolean) valuesInRow.get(j)); - break; - default: - LOGGER.error(JdbcMessages.NO_DATA_TYPE_MATCHED, columnType); - break; - } + COLUMN_VALUE_WRITER_SERVICE + .call(Type.fromTsDataType(columnType)) + .write(tsBlockBuilder.getColumnBuilder(j), valuesInRow.get(j)); } tsBlockBuilder.declarePosition(); } @@ -435,6 +430,11 @@ public static ByteBuffer convertTsBlock( } } + @FunctionalInterface + private interface ColumnValueWriter { + void write(ColumnBuilder builder, Object value); + } + protected void close(ResultSet rs, Statement stmt) { try { if (rs != null) { diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java index 75ff09a882e4..b4516efa7c81 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java @@ -29,14 +29,12 @@ import org.apache.thrift.TException; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BytesUtils; -import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.sql.Timestamp; import java.time.ZoneId; import java.util.ArrayList; @@ -156,38 +154,7 @@ public IoTDBJDBCDataSet( time = new byte[Long.BYTES]; currentBitmap = new byte[columnTypeDeduplicatedList.size()]; - values = new byte[columnTypeDeduplicatedList.size()][]; - for (int i = 0; i < values.length; i++) { - TSDataType dataType = columnTypeDeduplicatedList.get(i); - switch (dataType) { - case BOOLEAN: - values[i] = new byte[1]; - break; - case INT32: - case DATE: - values[i] = new byte[Integer.BYTES]; - break; - case INT64: - case TIMESTAMP: - values[i] = new byte[Long.BYTES]; - break; - case FLOAT: - values[i] = new byte[Float.BYTES]; - break; - case DOUBLE: - values[i] = new byte[Double.BYTES]; - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - values[i] = null; - break; - default: - throw new UnSupportedDataTypeException( - String.format(DATA_TYPE_NOT_SUPPORTED, columnTypeDeduplicatedList.get(i))); - } - } + values = initializeValueBuffers(columnTypeDeduplicatedList); this.tsQueryDataSet = queryDataSet; this.emptyResultSet = (queryDataSet == null || !queryDataSet.time.hasRemaining()); } @@ -275,38 +242,7 @@ public IoTDBJDBCDataSet( time = new byte[Long.BYTES]; currentBitmap = new byte[columnTypeDeduplicatedList.size()]; - values = new byte[columnTypeDeduplicatedList.size()][]; - for (int i = 0; i < values.length; i++) { - TSDataType dataType = columnTypeDeduplicatedList.get(i); - switch (dataType) { - case BOOLEAN: - values[i] = new byte[1]; - break; - case INT32: - case DATE: - values[i] = new byte[Integer.BYTES]; - break; - case INT64: - case TIMESTAMP: - values[i] = new byte[Long.BYTES]; - break; - case FLOAT: - values[i] = new byte[Float.BYTES]; - break; - case DOUBLE: - values[i] = new byte[Double.BYTES]; - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - values[i] = null; - break; - default: - throw new UnSupportedDataTypeException( - String.format(DATA_TYPE_NOT_SUPPORTED, columnTypeDeduplicatedList.get(i))); - } - } + values = initializeValueBuffers(columnTypeDeduplicatedList); this.tsQueryDataSet = queryDataSet; this.emptyResultSet = (queryDataSet == null || !queryDataSet.time.hasRemaining()); } @@ -421,26 +357,11 @@ public void constructOneRow() { if (!isNull(i, rowsIndex)) { ByteBuffer valueBuffer = tsQueryDataSet.valueList.get(i); TSDataType dataType = columnTypeDeduplicatedList.get(i); - switch (dataType) { - case BOOLEAN: - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - case DATE: - case TIMESTAMP: - valueBuffer.get(values[i]); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - int length = valueBuffer.getInt(); - values[i] = ReadWriteIOUtils.readBytes(valueBuffer, length); - break; - default: - throw new UnSupportedDataTypeException( - String.format(DATA_TYPE_NOT_SUPPORTED, columnTypeDeduplicatedList.get(i))); + if (dataType.isBinary()) { + int length = valueBuffer.getInt(); + values[i] = ReadWriteIOUtils.readBytes(valueBuffer, length); + } else { + valueBuffer.get(values[i]); } } } @@ -448,6 +369,34 @@ public void constructOneRow() { hasCachedRecord = true; } + static byte[][] initializeValueBuffers(List dataTypes) { + byte[][] valueBuffers = new byte[dataTypes.size()][]; + for (int i = 0; i < dataTypes.size(); i++) { + TSDataType dataType = dataTypes.get(i); + if (dataType == TSDataType.VECTOR || dataType == TSDataType.UNKNOWN) { + throw unsupportedDataType(dataType); + } + if (dataType.isBinary()) { + continue; + } + final int dataTypeSize; + try { + dataTypeSize = dataType.getDataTypeSize(); + } catch (UnSupportedDataTypeException e) { + throw unsupportedDataType(dataType); + } + if (dataTypeSize == 0) { + throw unsupportedDataType(dataType); + } + valueBuffers[i] = new byte[dataTypeSize]; + } + return valueBuffers; + } + + private static UnSupportedDataTypeException unsupportedDataType(TSDataType dataType) { + return new UnSupportedDataTypeException(String.format(DATA_TYPE_NOT_SUPPORTED, dataType)); + } + public boolean isNull(int columnIndex) throws StatementExecutionException { int index = columnOrdinalMap.get(findColumnNameByIndex(columnIndex)) - START_INDEX; // time column will never be null @@ -598,30 +547,9 @@ public String getValueByName(String columnName) throws StatementExecutionExcepti } public String getString(int index, TSDataType tsDataType, byte[][] values) { - switch (tsDataType) { - case BOOLEAN: - return String.valueOf(BytesUtils.bytesToBool(values[index])); - case INT32: - return String.valueOf(BytesUtils.bytesToInt(values[index])); - case INT64: - case TIMESTAMP: - return String.valueOf(BytesUtils.bytesToLong(values[index])); - case FLOAT: - return String.valueOf(BytesUtils.bytesToFloat(values[index])); - case DOUBLE: - return String.valueOf(BytesUtils.bytesToDouble(values[index])); - case TEXT: - case STRING: - return new String(values[index], StandardCharsets.UTF_8); - case OBJECT: - return BytesUtils.parseObjectByteArrayToString(values[index]); - case BLOB: - return BytesUtils.parseBlobByteArrayToString(values[index]); - case DATE: - return DateUtils.formatDate(BytesUtils.bytesToInt(values[index])); - default: - return null; - } + return TypeServices.JDBC_STRING_READER_SERVICE + .call(Type.fromTsDataType(tsDataType)) + .apply(values[index]); } public Object getObjectByName(String columnName) throws StatementExecutionException { @@ -639,30 +567,9 @@ public Object getObjectByName(String columnName) throws StatementExecutionExcept } public Object getObject(int index, TSDataType tsDataType, byte[][] values) { - switch (tsDataType) { - case BOOLEAN: - return BytesUtils.bytesToBool(values[index]); - case INT32: - return BytesUtils.bytesToInt(values[index]); - case INT64: - return BytesUtils.bytesToLong(values[index]); - case FLOAT: - return BytesUtils.bytesToFloat(values[index]); - case DOUBLE: - return BytesUtils.bytesToDouble(values[index]); - case TEXT: - case STRING: - return new String(values[index], StandardCharsets.UTF_8); - case OBJECT: - case BLOB: - return new Binary(values[index]); - case TIMESTAMP: - return new Timestamp(BytesUtils.bytesToLong(values[index])); - case DATE: - return DateUtils.parseIntToDate(BytesUtils.bytesToInt(values[index])); - default: - return null; - } + return TypeServices.JDBC_OBJECT_READER_SERVICE + .call(Type.fromTsDataType(tsDataType)) + .apply(values[index]); } public String findColumnNameByIndex(int columnIndex) throws StatementExecutionException { diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java index 56200399b4e6..32f5ea5d45a3 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java @@ -27,12 +27,12 @@ import org.apache.iotdb.service.rpc.thrift.TSFetchResultsResp; import org.apache.thrift.TException; -import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.TsBlockSerde; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.DateUtils; import java.nio.ByteBuffer; @@ -495,37 +495,10 @@ private Object getObjectByTsBlockIndex(int tsBlockColumnIndex) return null; } lastReadWasNull = false; - TSDataType tsDataType = getDataTypeByTsBlockColumnIndex(tsBlockColumnIndex); - switch (tsDataType) { - case BOOLEAN: - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - return curTsBlock.getColumn(tsBlockColumnIndex).getObject(tsBlockIndex); - case TIMESTAMP: - long timestamp = - (tsBlockColumnIndex == -1 - ? curTsBlock.getTimeByIndex(tsBlockIndex) - : curTsBlock.getColumn(tsBlockColumnIndex).getLong(tsBlockIndex)); - return convertToTimestamp(timestamp, timeFactor); - case TEXT: - case STRING: - return curTsBlock - .getColumn(tsBlockColumnIndex) - .getBinary(tsBlockIndex) - .getStringValue(TSFileConfig.STRING_CHARSET); - case OBJECT: - return BytesUtils.parseObjectByteArrayToString( - curTsBlock.getColumn(tsBlockColumnIndex).getBinary(tsBlockIndex).getValues()); - case BLOB: - return BytesUtils.parseBlobByteArrayToString( - curTsBlock.getColumn(tsBlockColumnIndex).getBinary(tsBlockIndex).getValues()); - case DATE: - return DateUtils.formatDate(curTsBlock.getColumn(tsBlockColumnIndex).getInt(tsBlockIndex)); - default: - return null; - } + Type type = Type.fromTsDataType(getDataTypeByTsBlockColumnIndex(tsBlockColumnIndex)); + return TypeServices.RPC_OBJECT_READER_SERVICE + .call(type) + .read(type, getColumnByTsBlockColumnIndex(tsBlockColumnIndex), tsBlockIndex, timeFactor); } public String getString(int columnIndex) throws StatementExecutionException { @@ -548,47 +521,22 @@ private String getStringByTsBlockColumnIndex(int tsBlockColumnIndex) return null; } lastReadWasNull = false; - return getString(tsBlockColumnIndex, getDataTypeByTsBlockColumnIndex(tsBlockColumnIndex)); - } - - private String getString(int index, TSDataType tsDataType) { - switch (tsDataType) { - case BOOLEAN: - return String.valueOf(curTsBlock.getColumn(index).getBoolean(tsBlockIndex)); - case INT32: - return String.valueOf(curTsBlock.getColumn(index).getInt(tsBlockIndex)); - case INT64: - return String.valueOf( - (index == -1 - ? curTsBlock.getTimeByIndex(tsBlockIndex) - : curTsBlock.getColumn(index).getLong(tsBlockIndex))); - case TIMESTAMP: - long timestamp = - (index == -1 - ? curTsBlock.getTimeByIndex(tsBlockIndex) - : curTsBlock.getColumn(index).getLong(tsBlockIndex)); - return RpcUtils.formatDatetime(timeFormat, timePrecision, timestamp, zoneId); - case FLOAT: - return String.valueOf(curTsBlock.getColumn(index).getFloat(tsBlockIndex)); - case DOUBLE: - return String.valueOf(curTsBlock.getColumn(index).getDouble(tsBlockIndex)); - case TEXT: - case STRING: - return curTsBlock - .getColumn(index) - .getBinary(tsBlockIndex) - .getStringValue(TSFileConfig.STRING_CHARSET); - case OBJECT: - return BytesUtils.parseObjectByteArrayToString( - curTsBlock.getColumn(index).getBinary(tsBlockIndex).getValues()); - case BLOB: - return BytesUtils.parseBlobByteArrayToString( - curTsBlock.getColumn(index).getBinary(tsBlockIndex).getValues()); - case DATE: - return DateUtils.formatDate(curTsBlock.getColumn(index).getInt(tsBlockIndex)); - default: - return null; - } + Type type = Type.fromTsDataType(getDataTypeByTsBlockColumnIndex(tsBlockColumnIndex)); + return TypeServices.RPC_STRING_READER_SERVICE + .call(type) + .read( + type, + getColumnByTsBlockColumnIndex(tsBlockColumnIndex), + tsBlockIndex, + timeFormat, + timePrecision, + zoneId); + } + + private Column getColumnByTsBlockColumnIndex(int tsBlockColumnIndex) { + return tsBlockColumnIndex == -1 + ? curTsBlock.getTimeColumn() + : curTsBlock.getColumn(tsBlockColumnIndex); } public Timestamp getTimestamp(int columnIndex) throws StatementExecutionException { diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TypeServices.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TypeServices.java new file mode 100644 index 000000000000..4b0311243723 --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TypeServices.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rpc; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; + +import java.nio.charset.StandardCharsets; +import java.sql.Timestamp; +import java.time.ZoneId; +import java.util.function.Function; + +final class TypeServices { + + static final TypeService> JDBC_STRING_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> value -> String.valueOf(BytesUtils.bytesToBool(value)); + case INT32 -> value -> String.valueOf(BytesUtils.bytesToInt(value)); + case INT64, TIMESTAMP -> value -> String.valueOf(BytesUtils.bytesToLong(value)); + case FLOAT -> value -> String.valueOf(BytesUtils.bytesToFloat(value)); + case DOUBLE -> value -> String.valueOf(BytesUtils.bytesToDouble(value)); + case TEXT, STRING -> value -> new String(value, StandardCharsets.UTF_8); + case OBJECT -> BytesUtils::parseObjectByteArrayToString; + case BLOB -> BytesUtils::parseBlobByteArrayToString; + case DATE -> value -> DateUtils.formatDate(BytesUtils.bytesToInt(value)); + case ROW, UNKNOWN, VECTOR -> ignored -> null; + }; + + static final TypeService> JDBC_OBJECT_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> BytesUtils::bytesToBool; + case INT32 -> BytesUtils::bytesToInt; + case INT64 -> BytesUtils::bytesToLong; + case FLOAT -> BytesUtils::bytesToFloat; + case DOUBLE -> BytesUtils::bytesToDouble; + case TEXT, STRING -> value -> new String(value, StandardCharsets.UTF_8); + case OBJECT, BLOB -> Binary::new; + case TIMESTAMP -> value -> new Timestamp(BytesUtils.bytesToLong(value)); + case DATE -> value -> DateUtils.parseIntToDate(BytesUtils.bytesToInt(value)); + case ROW, UNKNOWN, VECTOR -> ignored -> null; + }; + + static final TypeService RPC_OBJECT_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, INT64, FLOAT, DOUBLE -> + (actualType, column, position, timeFactor) -> + actualType.getObject(column, position); + case TIMESTAMP -> + (actualType, column, position, timeFactor) -> + RpcUtils.convertToTimestamp(actualType.getLong(column, position), timeFactor); + case TEXT, STRING -> + (actualType, column, position, timeFactor) -> + actualType + .getBinary(column, position) + .getStringValue(TSFileConfig.STRING_CHARSET); + case OBJECT -> + (actualType, column, position, timeFactor) -> + BytesUtils.parseObjectByteArrayToString( + actualType.getBinary(column, position).getValues()); + case BLOB -> + (actualType, column, position, timeFactor) -> + BytesUtils.parseBlobByteArrayToString( + actualType.getBinary(column, position).getValues()); + case DATE -> + (actualType, column, position, timeFactor) -> + DateUtils.formatDate(actualType.getInt(column, position)); + case ROW, UNKNOWN, VECTOR -> (actualType, column, position, timeFactor) -> null; + }; + + static final TypeService RPC_STRING_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (actualType, column, position, timeFormat, timePrecision, zoneId) -> + String.valueOf(actualType.getBoolean(column, position)); + case INT32 -> + (actualType, column, position, timeFormat, timePrecision, zoneId) -> + String.valueOf(actualType.getInt(column, position)); + case INT64 -> + (actualType, column, position, timeFormat, timePrecision, zoneId) -> + String.valueOf(actualType.getLong(column, position)); + case FLOAT -> + (actualType, column, position, timeFormat, timePrecision, zoneId) -> + String.valueOf(actualType.getFloat(column, position)); + case DOUBLE -> + (actualType, column, position, timeFormat, timePrecision, zoneId) -> + String.valueOf(actualType.getDouble(column, position)); + case TIMESTAMP -> + (actualType, column, position, timeFormat, timePrecision, zoneId) -> + RpcUtils.formatDatetime( + timeFormat, timePrecision, actualType.getLong(column, position), zoneId); + case TEXT, STRING -> + (actualType, column, position, timeFormat, timePrecision, zoneId) -> + actualType + .getBinary(column, position) + .getStringValue(TSFileConfig.STRING_CHARSET); + case OBJECT -> + (actualType, column, position, timeFormat, timePrecision, zoneId) -> + BytesUtils.parseObjectByteArrayToString( + actualType.getBinary(column, position).getValues()); + case BLOB -> + (actualType, column, position, timeFormat, timePrecision, zoneId) -> + BytesUtils.parseBlobByteArrayToString( + actualType.getBinary(column, position).getValues()); + case DATE -> + (actualType, column, position, timeFormat, timePrecision, zoneId) -> + DateUtils.formatDate(actualType.getInt(column, position)); + case ROW, UNKNOWN, VECTOR -> + (actualType, column, position, timeFormat, timePrecision, zoneId) -> null; + }; + + static { + JDBC_STRING_READER_SERVICE.check(); + JDBC_OBJECT_READER_SERVICE.check(); + RPC_OBJECT_READER_SERVICE.check(); + RPC_STRING_READER_SERVICE.check(); + } + + private TypeServices() {} + + @FunctionalInterface + interface RpcObjectReader { + + Object read(Type type, Column column, int position, int timeFactor); + } + + @FunctionalInterface + interface RpcStringReader { + + String read( + Type type, + Column column, + int position, + String timeFormat, + String timePrecision, + ZoneId zoneId); + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/stmt/PreparedParameterSerde.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/stmt/PreparedParameterSerde.java index e42710d23e12..6bf9cbba2265 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/stmt/PreparedParameterSerde.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/stmt/PreparedParameterSerde.java @@ -21,10 +21,13 @@ import org.apache.iotdb.rpc.i18n.RpcMessages; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; import java.io.IOException; import java.io.OutputStream; @@ -145,27 +148,25 @@ public static List deserialize(ByteBuffer buffer) { } private static Object deserializeValue(ByteBuffer buffer, TSDataType type) { - switch (type) { - case UNKNOWN: - return null; - case BOOLEAN: - return ReadWriteIOUtils.readBool(buffer); - case INT32: - return ReadWriteIOUtils.readInt(buffer); - case INT64: - return ReadWriteIOUtils.readLong(buffer); - case FLOAT: - return ReadWriteIOUtils.readFloat(buffer); - case DOUBLE: - return ReadWriteIOUtils.readDouble(buffer); - case TEXT: - case STRING: - return ReadWriteIOUtils.readString(buffer); - case BLOB: - return ReadWriteIOUtils.readBinary(buffer).getValues(); - default: - throw new IllegalArgumentException(RpcMessages.UNSUPPORTED_TYPE + type); + if (type == TSDataType.UNKNOWN) { + return null; + } + if (!type.isNumeric() + && type != TSDataType.BOOLEAN + && type != TSDataType.TEXT + && type != TSDataType.STRING + && type != TSDataType.BLOB) { + throw new IllegalArgumentException(RpcMessages.UNSUPPORTED_TYPE + type); + } + + TsPrimitiveType value = Type.fromTsDataType(type).deserialize(buffer); + if (type == TSDataType.BLOB) { + return value.getBinary().getValues(); + } + if (type == TSDataType.TEXT || type == TSDataType.STRING) { + return value.getBinary().getStringValue(TSFileConfig.STRING_CHARSET); } + return value.getValue(); } /** Convert byte array to hexadecimal string representation. */ diff --git a/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/IoTDBJDBCDataSetTest.java b/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/IoTDBJDBCDataSetTest.java new file mode 100644 index 000000000000..ccadd0d51d53 --- /dev/null +++ b/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/IoTDBJDBCDataSetTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rpc; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.UnknownType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.time.ZoneId; +import java.util.Arrays; + +public class IoTDBJDBCDataSetTest { + + @Test + public void testInitializeValueBuffersUsesDataTypeInterfaces() { + byte[][] buffers = + IoTDBJDBCDataSet.initializeValueBuffers( + Arrays.asList( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.DATE, + TSDataType.TIMESTAMP, + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB, + TSDataType.OBJECT)); + + Assert.assertEquals(Byte.BYTES, buffers[0].length); + Assert.assertEquals(Integer.BYTES, buffers[1].length); + Assert.assertEquals(Long.BYTES, buffers[2].length); + Assert.assertEquals(Float.BYTES, buffers[3].length); + Assert.assertEquals(Double.BYTES, buffers[4].length); + Assert.assertEquals(Integer.BYTES, buffers[5].length); + Assert.assertEquals(Long.BYTES, buffers[6].length); + Assert.assertNull(buffers[7]); + Assert.assertNull(buffers[8]); + Assert.assertNull(buffers[9]); + Assert.assertNull(buffers[10]); + } + + @Test + public void testInitializeValueBuffersRejectsInternalTypes() { + assertUnsupported(TSDataType.UNKNOWN); + assertUnsupported(TSDataType.VECTOR); + } + + @Test + public void testJdbcValueReadersUseTypeServices() { + Assert.assertEquals( + "42", + TypeServices.JDBC_STRING_READER_SERVICE + .call(Type.fromTsDataType(TSDataType.INT32)) + .apply(BytesUtils.intToBytes(42))); + Assert.assertEquals( + 42, + TypeServices.JDBC_OBJECT_READER_SERVICE + .call(Type.fromTsDataType(TSDataType.INT32)) + .apply(BytesUtils.intToBytes(42))); + + byte[] binary = new byte[] {1, 2, 3}; + Assert.assertEquals( + new Binary(binary), + TypeServices.JDBC_OBJECT_READER_SERVICE + .call(Type.fromTsDataType(TSDataType.BLOB)) + .apply(binary)); + + Assert.assertNull( + TypeServices.JDBC_STRING_READER_SERVICE.call(UnknownType.UNKNOWN).apply(new byte[0])); + Assert.assertNull( + TypeServices.JDBC_OBJECT_READER_SERVICE.call(UnknownType.UNKNOWN).apply(new byte[0])); + } + + @Test + public void testRpcValueReadersUseTypeServices() { + Type intType = Type.fromTsDataType(TSDataType.INT32); + Column intColumn = intType.createColumnBuilder(1).writeInt(42).build(); + Assert.assertEquals(42, readRpcObject(intType, intColumn, 1_000)); + Assert.assertEquals("42", readRpcString(intType, intColumn)); + + Type textType = Type.fromTsDataType(TSDataType.TEXT); + Column textColumn = + textType + .createColumnBuilder(1) + .writeBinary(new Binary("text", TSFileConfig.STRING_CHARSET)) + .build(); + Assert.assertEquals("text", readRpcObject(textType, textColumn, 1_000)); + Assert.assertEquals("text", readRpcString(textType, textColumn)); + + Type timestampType = Type.fromTsDataType(TSDataType.TIMESTAMP); + Column timestampColumn = timestampType.createColumnBuilder(1).writeLong(1_234_567).build(); + Assert.assertEquals( + RpcUtils.convertToTimestamp(1_234_567, 1_000_000), + readRpcObject(timestampType, timestampColumn, 1_000_000)); + Assert.assertEquals("1234567", readRpcString(timestampType, timestampColumn)); + + Type dateType = Type.fromTsDataType(TSDataType.DATE); + Column dateColumn = dateType.createColumnBuilder(1).writeInt(20240801).build(); + Assert.assertEquals(DateUtils.formatDate(20240801), readRpcObject(dateType, dateColumn, 1_000)); + Assert.assertEquals(DateUtils.formatDate(20240801), readRpcString(dateType, dateColumn)); + + byte[] blob = new byte[] {1, 2, 3}; + Type blobType = Type.fromTsDataType(TSDataType.BLOB); + Column blobColumn = blobType.createColumnBuilder(1).writeBinary(new Binary(blob)).build(); + Assert.assertEquals( + BytesUtils.parseBlobByteArrayToString(blob), readRpcObject(blobType, blobColumn, 1_000)); + Assert.assertEquals( + BytesUtils.parseBlobByteArrayToString(blob), readRpcString(blobType, blobColumn)); + + byte[] object = ByteBuffer.allocate(16).putLong(0).putLong(42).array(); + Type objectType = Type.fromTsDataType(TSDataType.OBJECT); + Column objectColumn = objectType.createColumnBuilder(1).writeBinary(new Binary(object)).build(); + Assert.assertEquals( + BytesUtils.parseObjectByteArrayToString(object), + readRpcObject(objectType, objectColumn, 1_000)); + Assert.assertEquals( + BytesUtils.parseObjectByteArrayToString(object), readRpcString(objectType, objectColumn)); + + Assert.assertNull( + TypeServices.RPC_OBJECT_READER_SERVICE + .call(UnknownType.UNKNOWN) + .read(UnknownType.UNKNOWN, null, 0, 1_000)); + Assert.assertNull( + TypeServices.RPC_STRING_READER_SERVICE + .call(UnknownType.UNKNOWN) + .read(UnknownType.UNKNOWN, null, 0, "long", "ms", ZoneId.of("UTC"))); + } + + private static Object readRpcObject(Type type, Column column, int timeFactor) { + return TypeServices.RPC_OBJECT_READER_SERVICE.call(type).read(type, column, 0, timeFactor); + } + + private static String readRpcString(Type type, Column column) { + return TypeServices.RPC_STRING_READER_SERVICE + .call(type) + .read(type, column, 0, "long", "ms", ZoneId.of("UTC")); + } + + private static void assertUnsupported(TSDataType dataType) { + UnSupportedDataTypeException exception = + Assert.assertThrows( + UnSupportedDataTypeException.class, + () -> IoTDBJDBCDataSet.initializeValueBuffers(Arrays.asList(dataType))); + Assert.assertTrue( + exception + .getMessage() + .endsWith(String.format(IoTDBJDBCDataSet.DATA_TYPE_NOT_SUPPORTED, dataType))); + } +} diff --git a/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/stmt/PreparedParameterSerdeTest.java b/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/stmt/PreparedParameterSerdeTest.java index cd3929c8224d..20de227def4b 100644 --- a/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/stmt/PreparedParameterSerdeTest.java +++ b/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/stmt/PreparedParameterSerdeTest.java @@ -33,6 +33,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; /** Unit tests for {@link PreparedParameterSerde}. */ @@ -124,4 +125,15 @@ public void testInvalidParameterCount() { buffer.flip(); deserialize(buffer); } + + @Test + public void testUnsupportedType() { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES + Byte.BYTES + Integer.BYTES); + buffer.putInt(1); + TSDataType.DATE.serializeTo(buffer); + buffer.putInt(20240801); + buffer.flip(); + + assertThrows(IllegalArgumentException.class, () -> deserialize(buffer)); + } } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index 39bb9c06cba5..0a8983070b77 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -70,10 +70,8 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -83,12 +81,10 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -108,6 +104,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.IntToLongFunction; import java.util.stream.Collectors; @SuppressWarnings({"java:S107", "java:S1135"}) // need enough parameters, ignore todos @@ -2504,11 +2501,7 @@ private TSInsertRecordsOfOneDeviceReq genTSInsertRecordsOfOneDeviceReq( if (!checkSorted(times)) { // sort - Integer[] index = new Integer[times.size()]; - for (int i = 0; i < times.size(); i++) { - index[i] = i; - } - Arrays.sort(index, Comparator.comparingLong(times::get)); + int[] index = sortedIndex(times); times.sort(Long::compareTo); // sort measurementList measurementsList = sortList(measurementsList, index); @@ -2561,11 +2554,7 @@ private TSInsertStringRecordsOfOneDeviceReq genTSInsertStringRecordsOfOneDeviceR } if (!checkSorted(times)) { - Integer[] index = new Integer[times.size()]; - for (int i = 0; i < index.length; i++) { - index[i] = i; - } - Arrays.sort(index, Comparator.comparingLong(times::get)); + int[] index = sortedIndex(times); times.sort(Long::compareTo); // sort measurementsList measurementsList = sortList(measurementsList, index); @@ -2592,7 +2581,7 @@ private TSInsertStringRecordsOfOneDeviceReq genTSInsertStringRecordsOfOneDeviceR * @param Input type * @return ordered list */ - private static List sortList(List source, Integer[] index) { + private static List sortList(List source, int[] index) { List sortedList = new ArrayList<>(index.length); for (int position : index) { sortedList.add(source.get(position)); @@ -3760,11 +3749,7 @@ public void sortTablet(Tablet tablet) { long[] timestamps = tablet.getTimestamps(); Object[] values = tablet.getValues(); BitMap[] bitMaps = tablet.getBitMaps(); - Integer[] index = new Integer[tablet.getRowSize()]; - for (int i = 0; i < tablet.getRowSize(); i++) { - index[i] = i; - } - Arrays.sort(index, Comparator.comparingLong(o -> timestamps[o])); + int[] index = sortedIndex(timestamps, tablet.getRowSize()); Arrays.sort(timestamps, 0, tablet.getRowSize()); int columnIndex = 0; for (int i = 0; i < tablet.getSchemas().size(); i++) { @@ -3798,64 +3783,8 @@ public void sortTablet(Tablet tablet) { * @param index index * @return sorted list */ - private Object sortList(Object valueList, TSDataType dataType, Integer[] index) { - switch (dataType) { - case BOOLEAN: - boolean[] boolValues = (boolean[]) valueList; - boolean[] sortedValues = new boolean[boolValues.length]; - for (int i = 0; i < index.length; i++) { - sortedValues[i] = boolValues[index[i]]; - } - return sortedValues; - case INT32: - int[] intValues = (int[]) valueList; - int[] sortedIntValues = new int[intValues.length]; - for (int i = 0; i < index.length; i++) { - sortedIntValues[i] = intValues[index[i]]; - } - return sortedIntValues; - case DATE: - LocalDate[] date = (LocalDate[]) valueList; - LocalDate[] sortedDateValues = new LocalDate[date.length]; - for (int i = 0; i < index.length; i++) { - sortedDateValues[i] = date[index[i]]; - } - return sortedDateValues; - case INT64: - case TIMESTAMP: - long[] longValues = (long[]) valueList; - long[] sortedLongValues = new long[longValues.length]; - for (int i = 0; i < index.length; i++) { - sortedLongValues[i] = longValues[index[i]]; - } - return sortedLongValues; - case FLOAT: - float[] floatValues = (float[]) valueList; - float[] sortedFloatValues = new float[floatValues.length]; - for (int i = 0; i < index.length; i++) { - sortedFloatValues[i] = floatValues[index[i]]; - } - return sortedFloatValues; - case DOUBLE: - double[] doubleValues = (double[]) valueList; - double[] sortedDoubleValues = new double[doubleValues.length]; - for (int i = 0; i < index.length; i++) { - sortedDoubleValues[i] = doubleValues[index[i]]; - } - return sortedDoubleValues; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary[] binaryValues = (Binary[]) valueList; - Binary[] sortedBinaryValues = new Binary[binaryValues.length]; - for (int i = 0; i < index.length; i++) { - sortedBinaryValues[i] = binaryValues[index[i]]; - } - return sortedBinaryValues; - default: - throw new UnSupportedDataTypeException(MSG_UNSUPPORTED_DATA_TYPE + dataType); - } + private Object sortList(Object valueList, TSDataType dataType, int[] index) { + return SessionUtils.sortValueList(valueList, dataType, index); } /** @@ -3865,7 +3794,7 @@ private Object sortList(Object valueList, TSDataType dataType, Integer[] index) * @param index index * @return sorted bitMap */ - private BitMap sortBitMap(BitMap bitMap, Integer[] index) { + private BitMap sortBitMap(BitMap bitMap, int[] index) { BitMap sortedBitMap = new BitMap(bitMap.getSize()); for (int i = 0; i < index.length; i++) { if (bitMap.isMarked(index[i])) { @@ -3875,6 +3804,52 @@ private BitMap sortBitMap(BitMap bitMap, Integer[] index) { return sortedBitMap; } + private static int[] sortedIndex(List values) { + return sortedIndex(values.size(), index -> values.get(index)); + } + + private static int[] sortedIndex(long[] values, int size) { + return sortedIndex(size, index -> values[index]); + } + + private static int[] sortedIndex(int size, IntToLongFunction valueProvider) { + int[] index = new int[size]; + int[] scratch = new int[size]; + for (int i = 0; i < size; i++) { + index[i] = i; + } + sortIndexes(index, scratch, 0, size, valueProvider); + return index; + } + + private static void sortIndexes( + int[] index, int[] scratch, int from, int to, IntToLongFunction valueProvider) { + if (to - from < 2) { + return; + } + int middle = (from + to) >>> 1; + sortIndexes(index, scratch, from, middle, valueProvider); + sortIndexes(index, scratch, middle, to, valueProvider); + + int left = from; + int right = middle; + int destination = from; + while (left < middle && right < to) { + if (valueProvider.applyAsLong(index[left]) <= valueProvider.applyAsLong(index[right])) { + scratch[destination++] = index[left++]; + } else { + scratch[destination++] = index[right++]; + } + } + while (left < middle) { + scratch[destination++] = index[left++]; + } + while (right < to) { + scratch[destination++] = index[right++]; + } + System.arraycopy(scratch, from, index, from, to - from); + } + @Override public void setSchemaTemplate(String templateName, String prefixPath) throws IoTDBConnectionException, StatementExecutionException { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionTypeServices.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionTypeServices.java new file mode 100644 index 000000000000..24119c3ca464 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionTypeServices.java @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.session.util; + +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.session.Session; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.encoder.Encoder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.record.Tablet; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.time.LocalDate; + +/** Type-specific operations used by the Session record-value wire format. */ +final class SessionTypeServices { + + private static final int EMPTY_DATE_INT = 10000101; + + private static final TypeService VALUE_LENGTH_CALCULATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> value -> Byte.BYTES; + case INT32, DATE -> value -> Integer.BYTES; + case INT64, TIMESTAMP -> value -> Long.BYTES; + case FLOAT -> value -> Float.BYTES; + case DOUBLE -> value -> Double.BYTES; + case TEXT, STRING, OBJECT -> value -> Integer.BYTES + getTextBytes(value).length; + case BLOB -> value -> Integer.BYTES + ((Binary) value).getValues().length; + case ROW, UNKNOWN, VECTOR -> + value -> { + throw unsupportedDataType(type.getTypeEnum()); + }; + }; + + private static final TypeService VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (value, buffer) -> ReadWriteIOUtils.write((Boolean) value, buffer); + case INT32 -> (value, buffer) -> ReadWriteIOUtils.write((Integer) value, buffer); + case DATE -> + (value, buffer) -> + ReadWriteIOUtils.write( + DateUtils.parseDateExpressionToInt((LocalDate) value), buffer); + case INT64, TIMESTAMP -> + (value, buffer) -> ReadWriteIOUtils.write((Long) value, buffer); + case FLOAT -> (value, buffer) -> ReadWriteIOUtils.write((Float) value, buffer); + case DOUBLE -> (value, buffer) -> ReadWriteIOUtils.write((Double) value, buffer); + case TEXT, STRING -> + (value, buffer) -> { + byte[] bytes = getTextBytes(value); + ReadWriteIOUtils.write(bytes.length, buffer); + buffer.put(bytes); + }; + case BLOB -> + (value, buffer) -> { + byte[] bytes = ((Binary) value).getValues(); + ReadWriteIOUtils.write(bytes.length, buffer); + buffer.put(bytes); + }; + // OBJECT was accepted by length calculation historically, but not by value writing. + case OBJECT, ROW, UNKNOWN, VECTOR -> + (value, buffer) -> { + throw unsupportedDataType(type.getTypeEnum()); + }; + }; + + private static final TypeService + TABLET_COLUMN_OCCUPATION_CALCULATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (values, columnIndex, rowSize) -> rowSize; + case INT32, FLOAT, DATE -> + (values, columnIndex, rowSize) -> rowSize * Integer.BYTES; + case INT64, DOUBLE, TIMESTAMP -> + (values, columnIndex, rowSize) -> rowSize * Long.BYTES; + case TEXT, BLOB, STRING, OBJECT -> + (values, columnIndex, rowSize) -> { + int occupation = rowSize * Integer.BYTES; + Binary[] binaries = (Binary[]) values[columnIndex]; + for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { + occupation += + binaries[rowIndex] != null + ? binaries[rowIndex].getLength() + : Binary.EMPTY_VALUE.getLength(); + } + return occupation; + }; + case ROW, UNKNOWN, VECTOR -> + (values, columnIndex, rowSize) -> { + throw unsupportedTabletDataType(type.getTypeEnum()); + }; + }; + + private static final TypeService TABLET_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (tablet, columnIndex, valueBuffer) -> { + int[] values = (int[]) tablet.getValues()[columnIndex]; + for (int index = 0; index < tablet.getRowSize(); index++) { + valueBuffer.putInt( + tablet.isNull(index, columnIndex) ? Integer.MIN_VALUE : values[index]); + } + }; + case INT64, TIMESTAMP -> + (tablet, columnIndex, valueBuffer) -> { + long[] values = (long[]) tablet.getValues()[columnIndex]; + for (int index = 0; index < tablet.getRowSize(); index++) { + valueBuffer.putLong( + tablet.isNull(index, columnIndex) ? Long.MIN_VALUE : values[index]); + } + }; + case FLOAT -> + (tablet, columnIndex, valueBuffer) -> { + float[] values = (float[]) tablet.getValues()[columnIndex]; + for (int index = 0; index < tablet.getRowSize(); index++) { + valueBuffer.putFloat( + tablet.isNull(index, columnIndex) ? Float.MIN_VALUE : values[index]); + } + }; + case DOUBLE -> + (tablet, columnIndex, valueBuffer) -> { + double[] values = (double[]) tablet.getValues()[columnIndex]; + for (int index = 0; index < tablet.getRowSize(); index++) { + valueBuffer.putDouble( + tablet.isNull(index, columnIndex) ? Double.MIN_VALUE : values[index]); + } + }; + case BOOLEAN -> + (tablet, columnIndex, valueBuffer) -> { + boolean[] values = (boolean[]) tablet.getValues()[columnIndex]; + for (int index = 0; index < tablet.getRowSize(); index++) { + valueBuffer.put( + BytesUtils.boolToByte(!tablet.isNull(index, columnIndex) && values[index])); + } + }; + case TEXT, STRING, BLOB, OBJECT -> + (tablet, columnIndex, valueBuffer) -> { + Binary[] values = (Binary[]) tablet.getValues()[columnIndex]; + for (int index = 0; index < tablet.getRowSize(); index++) { + Binary value = + !tablet.isNull(index, columnIndex) && values[index] != null + ? values[index] + : Binary.EMPTY_VALUE; + valueBuffer.putInt(value.getLength()); + valueBuffer.put(value.getValues()); + } + }; + case DATE -> + (tablet, columnIndex, valueBuffer) -> { + LocalDate[] values = (LocalDate[]) tablet.getValues()[columnIndex]; + for (int index = 0; index < tablet.getRowSize(); index++) { + valueBuffer.putInt( + !tablet.isNull(index, columnIndex) && values[index] != null + ? DateUtils.parseDateExpressionToInt(values[index]) + : EMPTY_DATE_INT); + } + }; + case ROW, UNKNOWN, VECTOR -> + (tablet, columnIndex, valueBuffer) -> { + throw unsupportedTabletDataType(type.getTypeEnum()); + }; + }; + + private static final TypeService TABLET_VALUE_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (tablet, columnIndex, encoder, outputStream) -> { + int[] values = (int[]) tablet.getValues()[columnIndex]; + int lastNonNullValue = 0; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, columnIndex)) { + lastNonNullValue = values[index]; + } + encoder.encode(lastNonNullValue, outputStream); + } + }; + case INT64, TIMESTAMP -> + (tablet, columnIndex, encoder, outputStream) -> { + long[] values = (long[]) tablet.getValues()[columnIndex]; + long lastNonNullValue = 0; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, columnIndex)) { + lastNonNullValue = values[index]; + } + encoder.encode(lastNonNullValue, outputStream); + } + }; + case FLOAT -> + (tablet, columnIndex, encoder, outputStream) -> { + float[] values = (float[]) tablet.getValues()[columnIndex]; + float lastNonNullValue = 0.0f; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, columnIndex)) { + lastNonNullValue = values[index]; + } + encoder.encode(lastNonNullValue, outputStream); + } + }; + case DOUBLE -> + (tablet, columnIndex, encoder, outputStream) -> { + double[] values = (double[]) tablet.getValues()[columnIndex]; + double lastNonNullValue = 0.0; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, columnIndex)) { + lastNonNullValue = values[index]; + } + encoder.encode(lastNonNullValue, outputStream); + } + }; + case BOOLEAN -> + (tablet, columnIndex, encoder, outputStream) -> { + boolean[] values = (boolean[]) tablet.getValues()[columnIndex]; + boolean lastNonNullValue = false; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, columnIndex)) { + lastNonNullValue = values[index]; + } + encoder.encode(lastNonNullValue, outputStream); + } + }; + case TEXT, STRING, BLOB -> + (tablet, columnIndex, encoder, outputStream) -> { + Binary[] values = (Binary[]) tablet.getValues()[columnIndex]; + Binary lastNonNullValue = Binary.EMPTY_VALUE; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, columnIndex) && values[index] != null) { + lastNonNullValue = values[index]; + } + encoder.encode(lastNonNullValue, outputStream); + } + }; + case DATE -> + (tablet, columnIndex, encoder, outputStream) -> { + LocalDate[] values = (LocalDate[]) tablet.getValues()[columnIndex]; + int lastNonNullValue = EMPTY_DATE_INT; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, columnIndex)) { + lastNonNullValue = DateUtils.parseDateExpressionToInt(values[index]); + } + // Previous values make null runs more compressible without changing the bitmap. + encoder.encode(lastNonNullValue, outputStream); + } + }; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (tablet, columnIndex, encoder, outputStream) -> { + throw unsupportedTabletDataType(type.getTypeEnum()); + }; + }; + + private static final TypeService VALUE_LIST_SORTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (valueList, index) -> { + boolean[] values = (boolean[]) valueList; + boolean[] sortedValues = new boolean[values.length]; + for (int i = 0; i < index.length; i++) { + sortedValues[i] = values[index[i]]; + } + return sortedValues; + }; + case INT32 -> + (valueList, index) -> { + int[] values = (int[]) valueList; + int[] sortedValues = new int[values.length]; + for (int i = 0; i < index.length; i++) { + sortedValues[i] = values[index[i]]; + } + return sortedValues; + }; + case DATE -> + (valueList, index) -> { + LocalDate[] values = (LocalDate[]) valueList; + LocalDate[] sortedValues = new LocalDate[values.length]; + for (int i = 0; i < index.length; i++) { + sortedValues[i] = values[index[i]]; + } + return sortedValues; + }; + case INT64, TIMESTAMP -> + (valueList, index) -> { + long[] values = (long[]) valueList; + long[] sortedValues = new long[values.length]; + for (int i = 0; i < index.length; i++) { + sortedValues[i] = values[index[i]]; + } + return sortedValues; + }; + case FLOAT -> + (valueList, index) -> { + float[] values = (float[]) valueList; + float[] sortedValues = new float[values.length]; + for (int i = 0; i < index.length; i++) { + sortedValues[i] = values[index[i]]; + } + return sortedValues; + }; + case DOUBLE -> + (valueList, index) -> { + double[] values = (double[]) valueList; + double[] sortedValues = new double[values.length]; + for (int i = 0; i < index.length; i++) { + sortedValues[i] = values[index[i]]; + } + return sortedValues; + }; + case TEXT, BLOB, STRING, OBJECT -> + (valueList, index) -> { + Binary[] values = (Binary[]) valueList; + Binary[] sortedValues = new Binary[values.length]; + for (int i = 0; i < index.length; i++) { + sortedValues[i] = values[index[i]]; + } + return sortedValues; + }; + case ROW, UNKNOWN, VECTOR -> + (valueList, index) -> { + throw unsupportedValueListDataType(type.getTypeEnum()); + }; + }; + + static { + VALUE_LENGTH_CALCULATOR_SERVICE.check(); + VALUE_WRITER_SERVICE.check(); + TABLET_COLUMN_OCCUPATION_CALCULATOR_SERVICE.check(); + TABLET_VALUE_WRITER_SERVICE.check(); + TABLET_VALUE_ENCODER_SERVICE.check(); + VALUE_LIST_SORTER_SERVICE.check(); + } + + private SessionTypeServices() {} + + static ValueLengthCalculator valueLengthCalculator(TSDataType dataType) { + return VALUE_LENGTH_CALCULATOR_SERVICE.call(Type.fromTsDataType(dataType)); + } + + static ValueWriter valueWriter(TSDataType dataType) { + return VALUE_WRITER_SERVICE.call(Type.fromTsDataType(dataType)); + } + + static TabletColumnOccupationCalculator tabletColumnOccupationCalculator(TSDataType dataType) { + return TABLET_COLUMN_OCCUPATION_CALCULATOR_SERVICE.call(Type.fromTsDataType(dataType)); + } + + static TabletValueWriter tabletValueWriter(TSDataType dataType) { + return TABLET_VALUE_WRITER_SERVICE.call(Type.fromTsDataType(dataType)); + } + + static TabletValueEncoder tabletValueEncoder(TSDataType dataType) { + return TABLET_VALUE_ENCODER_SERVICE.call(Type.fromTsDataType(dataType)); + } + + static ValueListSorter valueListSorter(TSDataType dataType) { + return VALUE_LIST_SORTER_SERVICE.call(Type.fromTsDataType(dataType)); + } + + private static byte[] getTextBytes(Object value) { + if (value instanceof Binary binary) { + return binary.getValues(); + } + return ((String) value).getBytes(TSFileConfig.STRING_CHARSET); + } + + private static IoTDBConnectionException unsupportedDataType(Object dataType) { + return new IoTDBConnectionException(Session.MSG_UNSUPPORTED_DATA_TYPE + dataType); + } + + private static UnSupportedDataTypeException unsupportedTabletDataType(Object dataType) { + return new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", dataType)); + } + + private static UnSupportedDataTypeException unsupportedValueListDataType(Object dataType) { + return new UnSupportedDataTypeException(Session.MSG_UNSUPPORTED_DATA_TYPE + dataType); + } + + @FunctionalInterface + interface ValueLengthCalculator { + int calculate(Object value) throws IoTDBConnectionException; + } + + @FunctionalInterface + interface ValueWriter { + void write(Object value, ByteBuffer buffer) throws IoTDBConnectionException; + } + + @FunctionalInterface + interface TabletColumnOccupationCalculator { + int calculate(Object[] values, int columnIndex, int rowSize); + } + + @FunctionalInterface + interface TabletValueWriter { + void write(Tablet tablet, int columnIndex, ByteBuffer valueBuffer); + } + + @FunctionalInterface + interface TabletValueEncoder { + void encode( + Tablet tablet, int columnIndex, Encoder encoder, ByteArrayOutputStream outputStream); + } + + @FunctionalInterface + interface ValueListSorter { + Object sort(Object valueList, int[] index); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java index b3b29f0be932..d6ff4d735ae5 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java @@ -24,17 +24,13 @@ import org.apache.iotdb.rpc.UrlUtils; import org.apache.iotdb.session.i18n.SessionMessages; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.encoding.encoder.Encoder; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; -import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.slf4j.Logger; @@ -43,17 +39,13 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.time.LocalDate; import java.util.ArrayList; import java.util.List; -import static org.apache.iotdb.session.Session.MSG_UNSUPPORTED_DATA_TYPE; - public class SessionUtils { private static final Logger LOGGER = LoggerFactory.getLogger(SessionUtils.class); private static final byte TYPE_NULL = -2; - private static final int EMPTY_DATE_INT = 10000101; public static ByteBuffer getTimeBuffer(Tablet tablet) { ByteBuffer timeBuffer = ByteBuffer.allocate(getTimeBytesSize(tablet)); @@ -119,40 +111,8 @@ private static int getTotalValueOccupation(Tablet tablet) { private static int calOccupationOfOneColumn( TSDataType dataType, Object[] values, int columnIndex, int rowSize) { - int valueOccupation = 0; - switch (dataType) { - case BOOLEAN: - valueOccupation += rowSize; - break; - case INT32: - case FLOAT: - case DATE: - valueOccupation += rowSize * 4; - break; - case INT64: - case DOUBLE: - case TIMESTAMP: - valueOccupation += rowSize * 8; - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueOccupation += rowSize * 4; - Binary[] binaries = (Binary[]) values[columnIndex]; - for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { - valueOccupation += - binaries[rowIndex] != null - ? binaries[rowIndex].getLength() - : Binary.EMPTY_VALUE.getLength(); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - SessionMessages.EXCEPTION_DATA_TYPE_ARG_NOT_SUPPORTED_31213160, dataType)); - } - return valueOccupation; + return SessionTypeServices.tabletColumnOccupationCalculator(dataType) + .calculate(values, columnIndex, rowSize); } public static ByteBuffer getValueBuffer( @@ -163,47 +123,17 @@ public static ByteBuffer getValueBuffer( return buffer; } + public static Object sortValueList(Object valueList, TSDataType dataType, int[] index) { + return SessionTypeServices.valueListSorter(dataType).sort(valueList, index); + } + public static int calculateLength(List types, List values) throws IoTDBConnectionException { int res = 0; for (int i = 0; i < types.size(); i++) { // types res += Byte.BYTES; - switch (types.get(i)) { - case BOOLEAN: - res += 1; - break; - case INT32: - case DATE: - res += Integer.BYTES; - break; - case INT64: - case TIMESTAMP: - res += Long.BYTES; - break; - case FLOAT: - res += Float.BYTES; - break; - case DOUBLE: - res += Double.BYTES; - break; - case TEXT: - case STRING: - case OBJECT: - res += Integer.BYTES; - if (values.get(i) instanceof Binary) { - res += ((Binary) values.get(i)).getValues().length; - } else { - res += ((String) values.get(i)).getBytes(TSFileConfig.STRING_CHARSET).length; - } - break; - case BLOB: - res += Integer.BYTES; - res += ((Binary) values.get(i)).getValues().length; - break; - default: - throw new IoTDBConnectionException(MSG_UNSUPPORTED_DATA_TYPE + types.get(i)); - } + res += SessionTypeServices.valueLengthCalculator(types.get(i)).calculate(values.get(i)); } return res; } @@ -228,47 +158,9 @@ public static void putValues( ReadWriteIOUtils.write(TYPE_NULL, buffer); continue; } - ReadWriteIOUtils.write(types.get(i), buffer); - switch (types.get(i)) { - case BOOLEAN: - ReadWriteIOUtils.write((Boolean) values.get(i), buffer); - break; - case INT32: - ReadWriteIOUtils.write((Integer) values.get(i), buffer); - break; - case DATE: - ReadWriteIOUtils.write( - DateUtils.parseDateExpressionToInt((LocalDate) values.get(i)), buffer); - break; - case INT64: - case TIMESTAMP: - ReadWriteIOUtils.write((Long) values.get(i), buffer); - break; - case FLOAT: - ReadWriteIOUtils.write((Float) values.get(i), buffer); - break; - case DOUBLE: - ReadWriteIOUtils.write((Double) values.get(i), buffer); - break; - case TEXT: - case STRING: - byte[] bytes; - if (values.get(i) instanceof Binary) { - bytes = ((Binary) values.get(i)).getValues(); - } else { - bytes = ((String) values.get(i)).getBytes(TSFileConfig.STRING_CHARSET); - } - ReadWriteIOUtils.write(bytes.length, buffer); - buffer.put(bytes); - break; - case BLOB: - bytes = ((Binary) values.get(i)).getValues(); - ReadWriteIOUtils.write(bytes.length, buffer); - buffer.put(bytes); - break; - default: - throw new IoTDBConnectionException(MSG_UNSUPPORTED_DATA_TYPE + types.get(i)); - } + TSDataType type = types.get(i); + ReadWriteIOUtils.write(type, buffer); + SessionTypeServices.valueWriter(type).write(values.get(i), buffer); } catch (Throwable e) { LOGGER.error( SessionMessages.LOG_CANNOT_PUT_VALUES_MEASUREMENT_ARG_TYPE_ARG_27AFC67B, @@ -281,99 +173,11 @@ public static void putValues( buffer.flip(); } - @SuppressWarnings({ - "squid:S6541", - "squid:S3776" - }) /// ignore Cognitive Complexity of methods should not be too high - // ignore Methods should not perform too many tasks (aka Brain method) private static void getValueBufferOfDataType( TSDataType dataType, Tablet tablet, int i, ByteBuffer valueBuffer) { - - switch (dataType) { - case INT32: - int[] intValues = (int[]) tablet.getValues()[i]; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i)) { - valueBuffer.putInt(intValues[index]); - } else { - valueBuffer.putInt(Integer.MIN_VALUE); - } - } - break; - case INT64: - case TIMESTAMP: - long[] longValues = (long[]) tablet.getValues()[i]; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i)) { - valueBuffer.putLong(longValues[index]); - } else { - valueBuffer.putLong(Long.MIN_VALUE); - } - } - break; - case FLOAT: - float[] floatValues = (float[]) tablet.getValues()[i]; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i)) { - valueBuffer.putFloat(floatValues[index]); - } else { - valueBuffer.putFloat(Float.MIN_VALUE); - } - } - break; - case DOUBLE: - double[] doubleValues = (double[]) tablet.getValues()[i]; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i)) { - valueBuffer.putDouble(doubleValues[index]); - } else { - valueBuffer.putDouble(Double.MIN_VALUE); - } - } - break; - case BOOLEAN: - boolean[] boolValues = (boolean[]) tablet.getValues()[i]; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i)) { - valueBuffer.put(BytesUtils.boolToByte(boolValues[index])); - } else { - valueBuffer.put(BytesUtils.boolToByte(false)); - } - } - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - Binary[] binaryValues = (Binary[]) tablet.getValues()[i]; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i) && binaryValues[index] != null) { - valueBuffer.putInt(binaryValues[index].getLength()); - valueBuffer.put(binaryValues[index].getValues()); - } else { - valueBuffer.putInt(Binary.EMPTY_VALUE.getLength()); - valueBuffer.put(Binary.EMPTY_VALUE.getValues()); - } - } - break; - case DATE: - LocalDate[] dateValues = (LocalDate[]) tablet.getValues()[i]; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i) && dateValues[index] != null) { - valueBuffer.putInt(DateUtils.parseDateExpressionToInt(dateValues[index])); - } else { - valueBuffer.putInt(EMPTY_DATE_INT); - } - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - SessionMessages.EXCEPTION_DATA_TYPE_ARG_NOT_SUPPORTED_31213160, dataType)); - } + SessionTypeServices.tabletValueWriter(dataType).write(tablet, i, valueBuffer); } - @SuppressWarnings({"java:S3776", "java:S6541"}) public static void encodeValue( TSDataType dataType, Tablet tablet, @@ -381,86 +185,7 @@ public static void encodeValue( Encoder encoder, ByteArrayOutputStream outputStream) { - switch (dataType) { - case INT32: - int[] intValues = (int[]) tablet.getValues()[i]; - int lastNonNullIntValue = 0; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i)) { - lastNonNullIntValue = intValues[index]; - } - encoder.encode(lastNonNullIntValue, outputStream); - } - break; - case INT64: - case TIMESTAMP: - long[] longValues = (long[]) tablet.getValues()[i]; - long lastNonNullLongValue = 0; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i)) { - lastNonNullLongValue = longValues[index]; - } - encoder.encode(lastNonNullLongValue, outputStream); - } - break; - case FLOAT: - float[] floatValues = (float[]) tablet.getValues()[i]; - float lastNonNullFloatValue = 0.0f; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i)) { - lastNonNullFloatValue = floatValues[index]; - } - encoder.encode(lastNonNullFloatValue, outputStream); - } - break; - case DOUBLE: - double[] doubleValues = (double[]) tablet.getValues()[i]; - double lastNonNullDoubleValue = 0.0; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i)) { - lastNonNullDoubleValue = doubleValues[index]; - } - encoder.encode(lastNonNullDoubleValue, outputStream); - } - break; - case BOOLEAN: - boolean[] boolValues = (boolean[]) tablet.getValues()[i]; - boolean lastNonNullBooleanValue = false; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i)) { - lastNonNullBooleanValue = boolValues[index]; - } - encoder.encode(lastNonNullBooleanValue, outputStream); - } - break; - case TEXT: - case STRING: - case BLOB: - Binary[] binaryValues = (Binary[]) tablet.getValues()[i]; - Binary lastNonNullBinaryValue = Binary.EMPTY_VALUE; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i) && binaryValues[index] != null) { - lastNonNullBinaryValue = binaryValues[index]; - } - encoder.encode(lastNonNullBinaryValue, outputStream); - } - break; - case DATE: - LocalDate[] dateValues = (LocalDate[]) tablet.getValues()[i]; - int lastNonNullDateValue = EMPTY_DATE_INT; - for (int index = 0; index < tablet.getRowSize(); index++) { - if (!tablet.isNull(index, i)) { - lastNonNullDateValue = DateUtils.parseDateExpressionToInt(dateValues[index]); - } - // use the previous value as the placeholder of nulls to increase encoding performance - encoder.encode(lastNonNullDateValue, outputStream); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - SessionMessages.EXCEPTION_DATA_TYPE_ARG_NOT_SUPPORTED_31213160, dataType)); - } + SessionTypeServices.tabletValueEncoder(dataType).encode(tablet, i, encoder, outputStream); try { encoder.flush(outputStream); } catch (IOException e) { diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java index b8524d4b3112..88b102642620 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java @@ -147,6 +147,27 @@ public void testGetValueBuffer2() throws IoTDBConnectionException { } } + // Covers DATE/TIMESTAMP conversion and STRING/BLOB payloads, including mixed fixed/variable + // widths; the encoded buffer must account for each type marker, length prefix, and payload. + @Test + public void testRecordValueTypeServices() throws IoTDBConnectionException { + List types = + Arrays.asList(TSDataType.DATE, TSDataType.TIMESTAMP, TSDataType.STRING, TSDataType.BLOB); + List values = + Arrays.asList( + LocalDate.of(2024, 4, 1), + 123L, + new Binary(new byte[] {1, 2}), + new Binary(new byte[] {3, 4, 5})); + List measurements = Arrays.asList("date", "timestamp", "string", "blob"); + + ByteBuffer buffer = SessionUtils.getValueBuffer(types, values, measurements); + + Assert.assertEquals( + 1 + Integer.BYTES + 1 + Long.BYTES + 1 + Integer.BYTES + 2 + 1 + Integer.BYTES + 3, + buffer.limit()); + } + @Test public void testGetValueBuffer3() { List schemas = new ArrayList<>(); diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionRecordHandler.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionRecordHandler.java index 8d658de88ed8..9cc63ff5f2f4 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionRecordHandler.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionRecordHandler.java @@ -27,19 +27,16 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.query.dataset.AbstractResultSet; import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.read.query.dataset.ResultSetMetadata; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; -import org.apache.tsfile.utils.DateUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import java.io.IOException; -import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -393,40 +390,9 @@ private TSRecord generateTsRecord( final String measurement = currentTablet.getSchemas().get(columnIndex).getMeasurementName(); final TSDataType dataType = currentTablet.getSchemas().get(columnIndex).getType(); final Object value = currentTablet.getValues()[columnIndex]; - switch (dataType) { - case BOOLEAN: - record.addPoint(measurement, ((boolean[]) value)[currentRowIndex]); - break; - case INT32: - record.addPoint(measurement, ((int[]) value)[currentRowIndex]); - break; - case DATE: - record.addPoint(measurement, ((LocalDate[]) value)[currentRowIndex]); - break; - case INT64: - case TIMESTAMP: - record.addPoint(measurement, ((long[]) value)[currentRowIndex]); - break; - case FLOAT: - record.addPoint(measurement, ((float[]) value)[currentRowIndex]); - break; - case DOUBLE: - record.addPoint(measurement, ((double[]) value)[currentRowIndex]); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - final Binary binary = ((Binary[]) value)[currentRowIndex]; - if (Objects.nonNull(binary)) { - record.addPoint(measurement, binary.getValues()); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - SubscriptionMessages.EXCEPTION_DATA_TYPE_ARG_NOT_SUPPORTED_31213160, dataType)); - } + TypeServices.TS_RECORD_VALUE_APPENDER_SERVICE + .call(Type.fromTsDataType(dataType)) + .append(record, measurement, value, currentRowIndex); } return record; } @@ -480,37 +446,9 @@ private static ColumnCategory convertColumnCategory( private static Field generateFieldFromTabletValue( final TSDataType dataType, final Object value, final int index) { final Field field = new Field(dataType); - switch (dataType) { - case BOOLEAN: - field.setBoolV(((boolean[]) value)[index]); - break; - case INT32: - field.setIntV(((int[]) value)[index]); - break; - case DATE: - field.setIntV(DateUtils.parseDateExpressionToInt(((LocalDate[]) value)[index])); - break; - case INT64: - case TIMESTAMP: - field.setLongV(((long[]) value)[index]); - break; - case FLOAT: - field.setFloatV(((float[]) value)[index]); - break; - case DOUBLE: - field.setDoubleV(((double[]) value)[index]); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - field.setBinaryV(new Binary((((Binary[]) value)[index]).getValues())); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - SubscriptionMessages.EXCEPTION_DATA_TYPE_ARG_NOT_SUPPORTED_31213160, dataType)); - } + TypeServices.FIELD_VALUE_READER_SERVICE + .call(Type.fromTsDataType(dataType)) + .read(field, value, index); return field; } diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/TypeServices.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/TypeServices.java new file mode 100644 index 000000000000..813d500ab70b --- /dev/null +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/TypeServices.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.session.subscription.payload; + +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.record.TSRecord; + +import java.time.LocalDate; +import java.util.Objects; + +/** + * Type-specific Tablet readers that preserve primitive-array access without intermediate boxing. + */ +final class TypeServices { + + static final TypeService TS_RECORD_VALUE_APPENDER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (record, measurement, values, rowIndex) -> + record.addPoint(measurement, ((boolean[]) values)[rowIndex]); + case INT32 -> + (record, measurement, values, rowIndex) -> + record.addPoint(measurement, ((int[]) values)[rowIndex]); + case DATE -> + (record, measurement, values, rowIndex) -> + record.addPoint(measurement, ((LocalDate[]) values)[rowIndex]); + case INT64, TIMESTAMP -> + (record, measurement, values, rowIndex) -> + record.addPoint(measurement, ((long[]) values)[rowIndex]); + case FLOAT -> + (record, measurement, values, rowIndex) -> + record.addPoint(measurement, ((float[]) values)[rowIndex]); + case DOUBLE -> + (record, measurement, values, rowIndex) -> + record.addPoint(measurement, ((double[]) values)[rowIndex]); + case TEXT, STRING, BLOB, OBJECT -> + (record, measurement, values, rowIndex) -> { + Binary binary = ((Binary[]) values)[rowIndex]; + if (Objects.nonNull(binary)) { + record.addPoint(measurement, binary.getValues()); + } + }; + case ROW, UNKNOWN, VECTOR -> + (record, measurement, values, rowIndex) -> { + throw unsupportedDataType(type.getTypeEnum()); + }; + }; + + static final TypeService FIELD_VALUE_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (field, values, index) -> field.setBoolV(((boolean[]) values)[index]); + case INT32 -> (field, values, index) -> field.setIntV(((int[]) values)[index]); + case DATE -> + (field, values, index) -> + field.setIntV( + DateUtils.parseDateExpressionToInt(((LocalDate[]) values)[index])); + case INT64, TIMESTAMP -> + (field, values, index) -> field.setLongV(((long[]) values)[index]); + case FLOAT -> (field, values, index) -> field.setFloatV(((float[]) values)[index]); + case DOUBLE -> (field, values, index) -> field.setDoubleV(((double[]) values)[index]); + case TEXT, STRING, BLOB, OBJECT -> + (field, values, index) -> + field.setBinaryV(new Binary(((Binary[]) values)[index].getValues())); + case ROW, UNKNOWN, VECTOR -> + (field, values, index) -> { + throw unsupportedDataType(type.getTypeEnum()); + }; + }; + + static { + TS_RECORD_VALUE_APPENDER_SERVICE.check(); + FIELD_VALUE_READER_SERVICE.check(); + } + + private TypeServices() {} + + private static UnSupportedDataTypeException unsupportedDataType(Object dataType) { + return new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", dataType)); + } + + @FunctionalInterface + interface TSRecordValueAppender { + + void append(TSRecord record, String measurement, Object values, int rowIndex); + } + + @FunctionalInterface + interface FieldValueReader { + + void read(Field field, Object values, int index); + } +} diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/payload/TypeServicesTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/payload/TypeServicesTest.java new file mode 100644 index 000000000000..718e4b653390 --- /dev/null +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/payload/TypeServicesTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.session.subscription.payload; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.write.record.TSRecord; +import org.junit.Assert; +import org.junit.Test; + +import java.time.LocalDate; + +public class TypeServicesTest { + + @Test + public void testAppendTabletValuesToTsRecord() { + TSRecord record = new TSRecord("root.sg.d", 1); + + TypeServices.TS_RECORD_VALUE_APPENDER_SERVICE + .call(Type.fromTsDataType(TSDataType.INT32)) + .append(record, "int", new int[] {42}, 0); + TypeServices.TS_RECORD_VALUE_APPENDER_SERVICE + .call(Type.fromTsDataType(TSDataType.DATE)) + .append(record, "date", new LocalDate[] {LocalDate.of(2024, 8, 1)}, 0); + TypeServices.TS_RECORD_VALUE_APPENDER_SERVICE + .call(Type.fromTsDataType(TSDataType.BLOB)) + .append(record, "nullBlob", new Binary[] {null}, 0); + + Assert.assertEquals(2, record.dataPointList.size()); + Assert.assertEquals(42, record.dataPointList.get(0).getValue()); + Assert.assertEquals(20240801, record.dataPointList.get(1).getValue()); + } + + @Test + public void testReadTabletValuesIntoField() { + Field dateField = new Field(TSDataType.DATE); + TypeServices.FIELD_VALUE_READER_SERVICE + .call(Type.fromTsDataType(TSDataType.DATE)) + .read(dateField, new LocalDate[] {LocalDate.of(2024, 8, 1)}, 0); + Assert.assertEquals( + DateUtils.parseDateExpressionToInt(LocalDate.of(2024, 8, 1)).intValue(), + dateField.getIntV()); + + Field blobField = new Field(TSDataType.BLOB); + TypeServices.FIELD_VALUE_READER_SERVICE + .call(Type.fromTsDataType(TSDataType.BLOB)) + .read(blobField, new Binary[] {new Binary(new byte[] {1, 2, 3})}, 0); + Assert.assertArrayEquals(new byte[] {1, 2, 3}, blobField.getBinaryV().getValues()); + } +} diff --git a/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java b/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java index d73526bd0e4c..750def8c4a3b 100644 --- a/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java +++ b/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java @@ -28,6 +28,7 @@ private CalcMessages() {} public static final String ARRAYS_NOT_SAME_LENGTH = "Arrays not same length"; public static final String CANNOT_ADD_NAN_TO_T_DIGEST = "Cannot add NaN to t-digest"; public static final String CANNOT_CAST_TO_BOOLEAN = "\"%s\" cannot be cast to [BOOLEAN]"; + public static final String CANNOT_CAST_TO_TYPE = "\"%s\" cannot be cast to [%s]"; public static final String CANNOT_PARSE_STRING_TO_DOUBLE = "Cannot parse String to double: "; public static final String CANT_HAPPEN_LOOP_FELL_THROUGH = "Can't happen ... loop fell through"; @@ -35,6 +36,20 @@ private CalcMessages() {} "CountAllAccumulator does not support statistics."; public static final String COUNT_IF_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS = "CountIfAccumulator does not support statistics"; + public static final String PERCENTILE_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS = + "PercentileAccumulator does not support statistics"; + public static final String PERCENTILE_REQUIRES_TWO_ARGUMENTS = + "PERCENTILE requires 2 arguments, but got %d"; + public static final String APPROX_PERCENTILE_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS = + "ApproxPercentileAccumulator does not support statistics"; + public static final String APPROX_COUNT_DISTINCT_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS = + "ApproxCountDistinctAccumulator does not support statistics"; + public static final String APPROX_MOST_FREQUENT_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS = + "ApproxMostFrequentAccumulator does not support statistics"; + public static final String NO_EXACT_DOUBLE_REPRESENTATION_FOR_LONG = + "no exact double representation for long: %s"; + public static final String APPROX_PERCENTILE_REQUIRES_TWO_OR_THREE_ARGUMENTS = + "APPROX_PERCENTILE requires 2 or 3 arguments, but got %d"; public static final String CURRENT_COLUMN_IS_NOT_OBJECT_COLUMN = "current column is not object column"; public static final String CURRENT_TS_BLOCK_SIZE_IS = "Current tsBlock size is : {}"; @@ -59,6 +74,8 @@ private CalcMessages() {} public static final String INITIAL_CAPACITY_EXCEEDS_LIMIT = "Initial capacity (%d) exceeds %d"; public static final String INPUT_ROW_UTILS_SHOULD_NOT_BE_INSTANTIATED = "InputRowUtils should not be instantiated."; + public static final String IN_LIST_LITERAL_FOR_TIMESTAMP_TYPE_RESTRICTION = + "InList Literal for TIMESTAMP can only be LongLiteral, DoubleLiteral and GenericLiteral, current is %s"; public static final String INVALID_AGGREGATION_FUNCTION = "Invalid Aggregation function: "; public static final String INVALID_TEXT_INPUT_FOR_BOOLEAN = "Invalid text input for boolean type: %s"; @@ -133,6 +150,42 @@ private CalcMessages() {} public static final String UNSUPPORTED_COMPARISON_OPERATOR = "Unsupported comparison operator: "; public static final String UNSUPPORTED_DATA_TYPE = "Unsupported data type: "; + public static final String UNSUPPORTED_DATA_TYPE_IN_AGGREGATION_AVG = + "Unsupported data type in aggregation AVG : %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_SUM_AGGREGATION = + "Unsupported data type in SUM Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_PERCENTILE_AGGREGATION = + "Unsupported data type in Percentile Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_APPROX_PERCENTILE_AGGREGATION = + "Unsupported data type in APPROX_PERCENTILE Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_APPROX_COUNT_DISTINCT_AGGREGATION = + "Unsupported data type in APPROX_COUNT_DISTINCT Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_FIRST_AGGREGATION = + "Unsupported data type in FIRST Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_LAST_AGGREGATION = + "Unsupported data type in LAST Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_MAX_AGGREGATION = + "Unsupported data type in MAX Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_MIN_AGGREGATION = + "Unsupported data type in MIN Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_FIRST_BY_AGGREGATION = + "Unsupported data type in FIRST_BY Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_LAST_BY_AGGREGATION = + "Unsupported data type in LAST_BY Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_EXTREME_AGGREGATION = + "Unsupported data type in EXTREME Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_MAX_BY_MIN_BY_AGGREGATION = + "Unsupported data type in MAX_BY/MIN_BY Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_MODE_AGGREGATION = + "Unsupported data type : %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_CENTRAL_MOMENT_AGGREGATION = + "Unsupported data type in CentralMoment Aggregation: %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_AGGREGATION_VARIANCE = + "Unsupported data type in aggregation variance : %s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_VARIANCE_AGGREGATION = + "Unsupported data type in VARIANCE Aggregation: %s"; + public static final String UNSUPPORTED_DEFAULT_VALUE_DATA_TYPE_IN_LAG = + "Unsupported default value's data type in Lag: "; public static final String UNSUPPORTED_DATA_TYPE_LOWER = "unsupported data type: "; public static final String UNSUPPORTED_FRAME_BOUND_TYPE = "Unsupported frame bound type: "; public static final String UNSUPPORTED_FUNCTION_KIND = "Unsupported function kind: "; diff --git a/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java b/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java index 07b3b549735a..06ac8d62cbbf 100644 --- a/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java +++ b/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java @@ -28,12 +28,27 @@ private CalcMessages() {} public static final String ARRAYS_NOT_SAME_LENGTH = "数组长度不一致"; public static final String CANNOT_ADD_NAN_TO_T_DIGEST = "不能将 NaN 添加到 t-digest"; public static final String CANNOT_CAST_TO_BOOLEAN = "\"%s\" 无法转换为 [BOOLEAN]"; + public static final String CANNOT_CAST_TO_TYPE = "\"%s\" 无法转换为 [%s]"; public static final String CANNOT_PARSE_STRING_TO_DOUBLE = "无法将字符串解析为 double："; public static final String CANT_HAPPEN_LOOP_FELL_THROUGH = "不应发生……循环穿透"; public static final String COUNT_ALL_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS = "CountAllAccumulator 不支持统计信息。"; public static final String COUNT_IF_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS = "CountIfAccumulator 不支持统计信息"; + public static final String PERCENTILE_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS = + "PercentileAccumulator 不支持统计信息"; + public static final String PERCENTILE_REQUIRES_TWO_ARGUMENTS = + "PERCENTILE 需要 2 个参数，但收到 %d 个参数"; + public static final String APPROX_PERCENTILE_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS = + "ApproxPercentileAccumulator 不支持统计信息"; + public static final String APPROX_COUNT_DISTINCT_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS = + "ApproxCountDistinctAccumulator 不支持统计信息"; + public static final String APPROX_MOST_FREQUENT_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS = + "ApproxMostFrequentAccumulator 不支持统计信息"; + public static final String NO_EXACT_DOUBLE_REPRESENTATION_FOR_LONG = + "long 无法精确表示为 double：%s"; + public static final String APPROX_PERCENTILE_REQUIRES_TWO_OR_THREE_ARGUMENTS = + "APPROX_PERCENTILE 需要 2 或 3 个参数，但收到 %d 个参数"; public static final String CURRENT_COLUMN_IS_NOT_OBJECT_COLUMN = "当前列不是对象列"; public static final String CURRENT_TS_BLOCK_SIZE_IS = "当前 tsBlock 大小为：{}"; public static final String DATA_TYPE_CANNOT_BE_ORDERED = "数据类型：%s 不能排序"; @@ -57,6 +72,8 @@ private CalcMessages() {} public static final String INITIAL_CAPACITY_EXCEEDS_LIMIT = "初始容量 (%d) 超过 %d"; public static final String INPUT_ROW_UTILS_SHOULD_NOT_BE_INSTANTIATED = "InputRowUtils 不应被实例化。"; + public static final String IN_LIST_LITERAL_FOR_TIMESTAMP_TYPE_RESTRICTION = + "TIMESTAMP 的 InList 字面量只能是 LongLiteral、DoubleLiteral 和 GenericLiteral，当前为 %s"; public static final String INVALID_AGGREGATION_FUNCTION = "无效的聚合函数："; public static final String INVALID_TEXT_INPUT_FOR_BOOLEAN = "布尔类型的文本输入无效：%s"; @@ -126,6 +143,42 @@ private CalcMessages() {} public static final String UNSUPPORTED_COLUMN_TRANSFORMER = "不支持的 ColumnTransformer"; public static final String UNSUPPORTED_COMPARISON_OPERATOR = "不支持的比较运算符："; public static final String UNSUPPORTED_DATA_TYPE = "不支持的数据类型："; + public static final String UNSUPPORTED_DATA_TYPE_IN_AGGREGATION_AVG = + "AVG 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_SUM_AGGREGATION = + "SUM 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_PERCENTILE_AGGREGATION = + "Percentile 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_APPROX_PERCENTILE_AGGREGATION = + "APPROX_PERCENTILE 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_APPROX_COUNT_DISTINCT_AGGREGATION = + "APPROX_COUNT_DISTINCT 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_FIRST_AGGREGATION = + "FIRST 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_LAST_AGGREGATION = + "LAST 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_MAX_AGGREGATION = + "MAX 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_MIN_AGGREGATION = + "MIN 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_FIRST_BY_AGGREGATION = + "FIRST_BY 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_LAST_BY_AGGREGATION = + "LAST_BY 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_EXTREME_AGGREGATION = + "EXTREME 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_MAX_BY_MIN_BY_AGGREGATION = + "MAX_BY/MIN_BY 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_MODE_AGGREGATION = + "不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_CENTRAL_MOMENT_AGGREGATION = + "CentralMoment 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_AGGREGATION_VARIANCE = + "variance 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DATA_TYPE_IN_VARIANCE_AGGREGATION = + "VARIANCE 聚合中不支持的数据类型：%s"; + public static final String UNSUPPORTED_DEFAULT_VALUE_DATA_TYPE_IN_LAG = + "Lag 中不支持的默认值数据类型："; public static final String UNSUPPORTED_DATA_TYPE_LOWER = "不支持的数据类型："; public static final String UNSUPPORTED_FRAME_BOUND_TYPE = "不支持的帧边界类型："; public static final String UNSUPPORTED_FUNCTION_KIND = "不支持的函数类型："; diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/CentralMomentAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/CentralMomentAccumulator.java index c8c0ab744682..d40fc14ab775 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/CentralMomentAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/CentralMomentAccumulator.java @@ -15,11 +15,13 @@ package org.apache.iotdb.calc.execution.aggregation; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; @@ -37,7 +39,7 @@ public enum MomentType { KURTOSIS } - private final TSDataType seriesDataType; + private final TypeServices.ColumnToDoubleConverter doubleValueConverter; private final MomentType momentType; private long count; @@ -47,7 +49,15 @@ public enum MomentType { private double m4; public CentralMomentAccumulator(TSDataType seriesDataType, MomentType momentType) { - this.seriesDataType = seriesDataType; + this.doubleValueConverter = + TypeServices.NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(Type.fromTsDataType(seriesDataType)) + .create( + () -> + new UnsupportedOperationException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_CENTRAL_MOMENT_AGGREGATION, + seriesDataType))); this.momentType = momentType; } @@ -62,26 +72,7 @@ public void addInput(Column[] columns, BitMap bitMap) { if (columns[1].isNull(i)) { continue; } - update(getDoubleValue(columns[1], i)); - } - } - - private double getDoubleValue(Column column, int position) { - switch (seriesDataType) { - case INT32: - case DATE: - return column.getInt(position); - case INT64: - case TIMESTAMP: - return column.getLong(position); - case FLOAT: - return column.getFloat(position); - case DOUBLE: - return column.getDouble(position); - default: - throw new UnsupportedOperationException( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_CENTRALMOMENT_AGGREGATION_0A19C2A8 - + seriesDataType); + update(doubleValueConverter.convert(columns[1], i)); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/CorrelationAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/CorrelationAccumulator.java index 97ad313c2b6d..527a169794a5 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/CorrelationAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/CorrelationAccumulator.java @@ -61,27 +61,15 @@ public void addInput(Column[] columns, BitMap bitMap) { continue; } - double x = getDoubleValue(columns[1], i, seriesDataTypes[0]); - double y = getDoubleValue(columns[2], i, seriesDataTypes[1]); + double x = getDoubleValue(columns[1], i); + double y = getDoubleValue(columns[2], i); update(x, y); } } - private double getDoubleValue(Column column, int position, TSDataType dataType) { - switch (dataType) { - case INT32: - return column.getInt(position); - case INT64: - case TIMESTAMP: - return column.getLong(position); - case FLOAT: - return column.getFloat(position); - case DOUBLE: - return column.getDouble(position); - default: - throw new IllegalArgumentException(CalcMessages.UNSUPPORTED_DATA_TYPE + dataType); - } + private double getDoubleValue(Column column, int position) { + return column.getDouble(position); } private void update(double x, double y) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/CovarianceAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/CovarianceAccumulator.java index dc2110ce098c..eb9bacdf44d0 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/CovarianceAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/CovarianceAccumulator.java @@ -65,26 +65,14 @@ public void addInput(Column[] columns, BitMap bitMap) { continue; } - double x = getDoubleValue(columns[1], i, seriesDataTypes[0]); - double y = getDoubleValue(columns[2], i, seriesDataTypes[1]); + double x = getDoubleValue(columns[1], i); + double y = getDoubleValue(columns[2], i); update(x, y); } } - private double getDoubleValue(Column column, int position, TSDataType dataType) { - switch (dataType) { - case INT32: - return column.getInt(position); - case INT64: - case TIMESTAMP: - return column.getLong(position); - case FLOAT: - return column.getFloat(position); - case DOUBLE: - return column.getDouble(position); - default: - throw new IllegalArgumentException(CalcMessages.UNSUPPORTED_DATA_TYPE + dataType); - } + private double getDoubleValue(Column column, int position) { + return column.getDouble(position); } private void update(double x, double y) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/RegressionAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/RegressionAccumulator.java index ca58ba7a68fb..99eaaf83782c 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/RegressionAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/RegressionAccumulator.java @@ -62,27 +62,15 @@ public void addInput(Column[] columns, BitMap bitMap) { continue; } - double y = getDoubleValue(columns[1], i, seriesDataTypes[0]); - double x = getDoubleValue(columns[2], i, seriesDataTypes[1]); + double y = getDoubleValue(columns[1], i); + double x = getDoubleValue(columns[2], i); update(x, y); } } - private double getDoubleValue(Column column, int position, TSDataType dataType) { - switch (dataType) { - case INT32: - return column.getInt(position); - case INT64: - case TIMESTAMP: - return column.getLong(position); - case FLOAT: - return column.getFloat(position); - case DOUBLE: - return column.getDouble(position); - default: - throw new IllegalArgumentException(CalcMessages.UNSUPPORTED_DATA_TYPE + dataType); - } + private double getDoubleValue(Column column, int position) { + return column.getDouble(position); } private void update(double x, double y) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/VarianceAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/VarianceAccumulator.java index 528f12982724..94ff849463e9 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/VarianceAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/aggregation/VarianceAccumulator.java @@ -20,11 +20,13 @@ package org.apache.iotdb.calc.execution.aggregation; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; @@ -43,6 +45,7 @@ public enum VarianceType { } private final TSDataType seriesDataType; + private final TypeServices.ColumnToDoubleConverter doubleValueConverter; private final VarianceType varianceType; @@ -52,36 +55,33 @@ public enum VarianceType { public VarianceAccumulator(TSDataType seriesDataType, VarianceType varianceType) { this.seriesDataType = seriesDataType; + this.doubleValueConverter = + TypeServices.NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(Type.fromTsDataType(seriesDataType)) + .create( + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_AGGREGATION_VARIANCE, + seriesDataType))); this.varianceType = varianceType; } @Override public void addInput(Column[] columns, BitMap bitMap) { - switch (seriesDataType) { - case INT32: - addIntInput(columns, bitMap); - return; - case INT64: - addLongInput(columns, bitMap); - return; - case FLOAT: - addFloatInput(columns, bitMap); - return; - case DOUBLE: - addDoubleInput(columns, bitMap); - return; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case STRING: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_AGGREGATION_VARIANCE_ARG_17A45959, - seriesDataType)); + checkInputDataType(); + int size = columns[0].getPositionCount(); + for (int i = 0; i < size; i++) { + if (bitMap != null && !bitMap.isMarked(i)) { + continue; + } + if (!columns[1].isNull(i)) { + double value = doubleValueConverter.convert(columns[1], i); + count++; + double delta = value - mean; + mean += delta / count; + m2 += delta * (value - mean); + } } } @@ -224,67 +224,11 @@ public TSDataType getFinalType() { return TSDataType.DOUBLE; } - private void addIntInput(Column[] columns, BitMap bitmap) { - int size = columns[0].getPositionCount(); - for (int i = 0; i < size; i++) { - if (bitmap != null && !bitmap.isMarked(i)) { - continue; - } - if (!columns[1].isNull(i)) { - int value = columns[1].getInt(i); - count++; - double delta = value - mean; - mean += delta / count; - m2 += delta * (value - mean); - } - } - } - - private void addLongInput(Column[] columns, BitMap bitmap) { - int size = columns[0].getPositionCount(); - for (int i = 0; i < size; i++) { - if (bitmap != null && !bitmap.isMarked(i)) { - continue; - } - if (!columns[1].isNull(i)) { - long value = columns[1].getLong(i); - count++; - double delta = value - mean; - mean += delta / count; - m2 += delta * (value - mean); - } - } - } - - private void addFloatInput(Column[] columns, BitMap bitmap) { - int size = columns[0].getPositionCount(); - for (int i = 0; i < size; i++) { - if (bitmap != null && !bitmap.isMarked(i)) { - continue; - } - if (!columns[1].isNull(i)) { - float value = columns[1].getFloat(i); - count++; - double delta = value - mean; - mean += delta / count; - m2 += delta * (value - mean); - } - } - } - - private void addDoubleInput(Column[] columns, BitMap bitmap) { - int size = columns[0].getPositionCount(); - for (int i = 0; i < size; i++) { - if (bitmap != null && !bitmap.isMarked(i)) { - continue; - } - if (!columns[1].isNull(i)) { - double value = columns[1].getDouble(i); - count++; - double delta = value - mean; - mean += delta / count; - m2 += delta * (value - mean); - } + private void checkInputDataType() { + if (!seriesDataType.isNumeric()) { + throw new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_AGGREGATION_VARIANCE, seriesDataType)); } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/TopKOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/TopKOperator.java index b2754cbbdd67..f2b527c1ebfc 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/TopKOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/TopKOperator.java @@ -22,7 +22,7 @@ import org.apache.iotdb.calc.execution.filter.TopKRuntimeFilter; import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; import org.apache.iotdb.calc.execution.operator.Operator; -import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.calc.utils.datastructure.MergeSortHeap; import org.apache.iotdb.calc.utils.datastructure.MergeSortKey; import org.apache.iotdb.calc.utils.datastructure.SortKey; @@ -35,20 +35,12 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; -import org.apache.tsfile.read.common.block.column.BinaryColumn; -import org.apache.tsfile.read.common.block.column.BooleanColumn; -import org.apache.tsfile.read.common.block.column.DoubleColumn; -import org.apache.tsfile.read.common.block.column.FloatColumn; -import org.apache.tsfile.read.common.block.column.IntColumn; -import org.apache.tsfile.read.common.block.column.LongColumn; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.util.ArrayList; import java.util.Comparator; import java.util.List; -import java.util.Optional; import java.util.concurrent.TimeUnit; import static com.google.common.util.concurrent.Futures.successfulAsList; @@ -308,61 +300,7 @@ private void initResultTsBlock() { int positionCount = topValue; Column[] columns = new Column[dataTypes.size()]; for (int i = 0; i < dataTypes.size(); i++) { - switch (dataTypes.get(i)) { - case BOOLEAN: - columns[i] = - new BooleanColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new boolean[positionCount]); - break; - case INT32: - columns[i] = - new IntColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new int[positionCount], - TSDataType.INT32); - break; - case DATE: - columns[i] = - new IntColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new int[positionCount], - TSDataType.DATE); - break; - case INT64: - case TIMESTAMP: - columns[i] = - new LongColumn( - positionCount, Optional.of(new boolean[positionCount]), new long[positionCount]); - break; - case FLOAT: - columns[i] = - new FloatColumn( - positionCount, Optional.of(new boolean[positionCount]), new float[positionCount]); - break; - case DOUBLE: - columns[i] = - new DoubleColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new double[positionCount]); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - columns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - break; - default: - throw new UnSupportedDataTypeException(CalcMessages.UNKNOWN_DATATYPE + dataTypes.get(i)); - } + columns[i] = Type.fromTsDataType(dataTypes.get(i)).createColumnWithMaxPosition(positionCount); } this.tmpResultTsBlock = constrcutResultTsBlock(positionCount, columns); } @@ -412,30 +350,9 @@ private TsBlock getResultFromCachedTopKResult() { private long getMemoryUsageOfOneMergeSortKey() { long memory = 0; for (TSDataType dataType : dataTypes) { - switch (dataType) { - case BOOLEAN: - memory += 1; - break; - case INT32: - case FLOAT: - case DATE: - memory += 4; - break; - case INT64: - case DOUBLE: - case VECTOR: - case TIMESTAMP: - memory += 8; - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - memory += 16; - break; - default: - throw new UnSupportedDataTypeException(CalcMessages.UNKNOWN_DATATYPE + dataType); - } + memory += + TypeServices.MEMORY_USAGE_OF_ONE_MERGE_SORT_KEY_SERVICE.call( + Type.fromTsDataType(dataType)); } return memory; } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/join/merge/MergeSortComparator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/join/merge/MergeSortComparator.java index 032fc5f39885..cc56514bde3e 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/join/merge/MergeSortComparator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/join/merge/MergeSortComparator.java @@ -19,12 +19,13 @@ package org.apache.iotdb.calc.execution.operator.process.join.merge; -import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.calc.utils.datastructure.SortKey; import org.apache.iotdb.commons.queryengine.plan.relational.planner.SortOrder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.collections4.comparators.ComparatorChain; +import org.apache.tsfile.read.common.type.Type; import java.util.ArrayList; import java.util.Comparator; @@ -65,47 +66,8 @@ public static Comparator getComparatorForTable( } public static Comparator getComparator(TSDataType dataType, int index, boolean asc) { - Comparator comparator; - switch (dataType) { - case INT32: - case DATE: - comparator = - Comparator.comparingInt( - (SortKey sortKey) -> sortKey.tsBlock.getColumn(index).getInt(sortKey.rowIndex)); - break; - case INT64: - case TIMESTAMP: - comparator = - Comparator.comparingLong( - (SortKey sortKey) -> sortKey.tsBlock.getColumn(index).getLong(sortKey.rowIndex)); - break; - case FLOAT: - comparator = - Comparator.comparingDouble( - (SortKey sortKey) -> sortKey.tsBlock.getColumn(index).getFloat(sortKey.rowIndex)); - break; - case DOUBLE: - comparator = - Comparator.comparingDouble( - (SortKey sortKey) -> sortKey.tsBlock.getColumn(index).getDouble(sortKey.rowIndex)); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - comparator = - Comparator.comparing( - (SortKey sortKey) -> sortKey.tsBlock.getColumn(index).getBinary(sortKey.rowIndex)); - break; - case BOOLEAN: - comparator = - Comparator.comparing( - (SortKey sortKey) -> sortKey.tsBlock.getColumn(index).getBoolean(sortKey.rowIndex)); - break; - default: - throw new IllegalArgumentException( - String.format(CalcMessages.DATA_TYPE_CANNOT_BE_ORDERED, dataType)); - } + Comparator comparator = + TypeServices.MERGE_SORT_COMPARATOR_SERVICE.call(Type.fromTsDataType(dataType)).apply(index); if (!asc) { comparator = comparator.reversed(); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java index 841fe965c193..3b55c8669118 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java @@ -19,7 +19,7 @@ package org.apache.iotdb.calc.execution.operator.process.join.merge.comparator; -import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.read.common.type.Type; @@ -59,38 +59,6 @@ public static List getAsofComparators( } public static JoinKeyComparator getComparator(Type type, boolean isAscending) { - switch (type.getTypeEnum()) { - case INT32: - case DATE: - return isAscending - ? AscIntTypeJoinKeyComparator.getInstance() - : DescIntTypeJoinKeyComparator.getInstance(); - case INT64: - case TIMESTAMP: - return isAscending - ? AscLongTypeJoinKeyComparator.getInstance() - : DescLongTypeJoinKeyComparator.getInstance(); - case FLOAT: - return isAscending - ? AscFloatTypeJoinKeyComparator.getInstance() - : DescFloatTypeJoinKeyComparator.getInstance(); - case DOUBLE: - return isAscending - ? AscDoubleTypeJoinKeyComparator.getInstance() - : DescDoubleTypeJoinKeyComparator.getInstance(); - case BOOLEAN: - return isAscending - ? AscBooleanTypeJoinKeyComparator.getInstance() - : DescBooleanTypeJoinKeyComparator.getInstance(); - case STRING: - case BLOB: - case TEXT: - return isAscending - ? AscBinaryTypeJoinKeyComparator.getInstance() - : DescBinaryTypeJoinKeyComparator.getInstance(); - default: - // other types are not supported. - throw new UnsupportedOperationException(CalcMessages.UNSUPPORTED_DATA_TYPE + type); - } + return TypeServices.JOIN_KEY_COMPARATOR_SERVICE.call(type).apply(isAscending); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/function/value/LagFunction.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/function/value/LagFunction.java index b8711824f913..d354ded68aa3 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/function/value/LagFunction.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/function/value/LagFunction.java @@ -20,11 +20,8 @@ package org.apache.iotdb.calc.execution.operator.process.window.function.value; import org.apache.iotdb.calc.execution.operator.process.window.partition.Partition; -import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.util.List; @@ -82,39 +79,6 @@ public void transform( } } - private void writeDefaultValue( - Partition partition, int defaultValChannel, int index, ColumnBuilder builder) { - TSDataType dataType = builder.getDataType(); - switch (dataType) { - case INT32: - case DATE: - builder.writeInt(partition.getInt(defaultValChannel, index)); - return; - case INT64: - case TIMESTAMP: - builder.writeLong(partition.getLong(defaultValChannel, index)); - return; - case FLOAT: - builder.writeFloat(partition.getFloat(defaultValChannel, index)); - return; - case DOUBLE: - builder.writeDouble(partition.getDouble(defaultValChannel, index)); - return; - case BOOLEAN: - builder.writeBoolean(partition.getBoolean(defaultValChannel, index)); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - builder.writeBinary(partition.getBinary(defaultValChannel, index)); - return; - default: - throw new UnSupportedDataTypeException( - CalcMessages.EXCEPTION_UNSUPPORTED_DEFAULT_VALUE_S_DATA_TYPE_LAG_BEB00511 + dataType); - } - } - @Override public boolean needFrame() { return false; diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/function/value/LeadFunction.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/function/value/LeadFunction.java index 1e9e1831c6aa..0430034023ff 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/function/value/LeadFunction.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/function/value/LeadFunction.java @@ -20,11 +20,8 @@ package org.apache.iotdb.calc.execution.operator.process.window.function.value; import org.apache.iotdb.calc.execution.operator.process.window.partition.Partition; -import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.util.List; @@ -83,39 +80,6 @@ public void transform( } } - private void writeDefaultValue( - Partition partition, int defaultValChannel, int index, ColumnBuilder builder) { - TSDataType dataType = builder.getDataType(); - switch (dataType) { - case INT32: - case DATE: - builder.writeInt(partition.getInt(defaultValChannel, index)); - return; - case INT64: - case TIMESTAMP: - builder.writeLong(partition.getLong(defaultValChannel, index)); - return; - case FLOAT: - builder.writeFloat(partition.getFloat(defaultValChannel, index)); - return; - case DOUBLE: - builder.writeDouble(partition.getDouble(defaultValChannel, index)); - return; - case BOOLEAN: - builder.writeBoolean(partition.getBoolean(defaultValChannel, index)); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - builder.writeBinary(partition.getBinary(defaultValChannel, index)); - return; - default: - throw new UnSupportedDataTypeException( - CalcMessages.EXCEPTION_UNSUPPORTED_DEFAULT_VALUE_S_DATA_TYPE_LAG_BEB00511 + dataType); - } - } - @Override public boolean needFrame() { return false; diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/function/value/ValueWindowFunction.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/function/value/ValueWindowFunction.java index daf30551e49c..d9ba44ac71a0 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/function/value/ValueWindowFunction.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/function/value/ValueWindowFunction.java @@ -21,8 +21,10 @@ import org.apache.iotdb.calc.execution.operator.process.window.function.WindowFunction; import org.apache.iotdb.calc.execution.operator.process.window.partition.Partition; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; public abstract class ValueWindowFunction implements WindowFunction { @Override @@ -45,6 +47,13 @@ public void transform( public abstract void transform( Partition partition, ColumnBuilder builder, int index, int frameStart, int frameEnd); + protected void writeDefaultValue( + Partition partition, int defaultValChannel, int index, ColumnBuilder builder) { + TypeServices.DEFAULT_VALUE_WRITER_SERVICE + .call(Type.fromTsDataType(builder.getDataType())) + .write(partition, defaultValChannel, index, builder); + } + @Override public boolean needPeerGroup() { return false; diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/partition/frame/RangeFrame.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/partition/frame/RangeFrame.java index cf59324bdd99..6e53ce6549d6 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/partition/frame/RangeFrame.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/partition/frame/RangeFrame.java @@ -24,10 +24,11 @@ import org.apache.iotdb.calc.execution.operator.process.window.utils.Range; import org.apache.iotdb.calc.execution.operator.process.window.utils.RowComparator; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.read.common.type.Type; import java.util.List; @@ -44,6 +45,7 @@ public class RangeFrame implements Frame { private List allSortedColumns; private ColumnList column; private TSDataType dataType; + private TypeServices.RangeFrameComparator rangeFrameComparator; private int partitionSize; private RowComparator peerGroupComparator; @@ -69,6 +71,8 @@ public RangeFrame( this.allSortedColumns = sortedColumns; this.column = sortedColumns.get(0); this.dataType = column.getDataType(); + this.rangeFrameComparator = + TypeServices.RANGE_FRAME_COMPARATOR_SERVICE.call(Type.fromTsDataType(dataType)); this.peerGroupComparator = comparator; this.recentRange = new Range(0, 0); } @@ -285,34 +289,14 @@ private int getAscFrameStartFollowing(int currentIndex, int recentIndex) { private boolean compareInAscFrameStartFollowing(int currentIndex, int recentIndex, int channel) { checkArgument(!partition.isNull(channel, currentIndex)); - switch (column.getDataType()) { - case INT32: - case DATE: - int currentInt = column.getInt(currentIndex); - int followInt = column.getInt(recentIndex); - int deltaInt = partition.getInt(channel, currentIndex); - return followInt >= currentInt + deltaInt; - case INT64: - case TIMESTAMP: - long currentLong = column.getLong(currentIndex); - long followLong = column.getLong(recentIndex); - long deltaLong = partition.getLong(channel, currentIndex); - return followLong >= currentLong + deltaLong; - case FLOAT: - float currentFloat = column.getFloat(currentIndex); - float followFloat = column.getFloat(recentIndex); - float deltaFloat = partition.getFloat(channel, currentIndex); - return followFloat >= currentFloat + deltaFloat; - case DOUBLE: - double currentDouble = column.getDouble(currentIndex); - double followDouble = column.getDouble(recentIndex); - double deltaDouble = partition.getDouble(channel, currentIndex); - return followDouble >= currentDouble + deltaDouble; - default: - // Unreachable - throw new UnSupportedDataTypeException( - CalcMessages.UNSUPPORTED_DATA_TYPE + column.getDataType()); - } + return rangeFrameComparator.compare( + column, + partition, + currentIndex, + recentIndex, + channel, + TypeServices.RangeFrameOffsetOperation.ADD, + TypeServices.RangeFrameComparison.GREATER_THAN_OR_EQUAL); } // Find first row which satisfy: @@ -331,34 +315,14 @@ private int getAscFrameEndFollowing(int currentIndex, int recentIndex) { private boolean compareInAscFrameEndFollowing(int currentIndex, int recentIndex, int channel) { checkArgument(!partition.isNull(channel, currentIndex)); - switch (column.getDataType()) { - case INT32: - case DATE: - int currentInt = column.getInt(currentIndex); - int followInt = column.getInt(recentIndex); - int deltaInt = partition.getInt(channel, currentIndex); - return followInt > currentInt + deltaInt; - case INT64: - case TIMESTAMP: - long currentLong = column.getLong(currentIndex); - long followLong = column.getLong(recentIndex); - long deltaLong = partition.getLong(channel, currentIndex); - return followLong > currentLong + deltaLong; - case FLOAT: - float currentFloat = column.getFloat(currentIndex); - float followFloat = column.getFloat(recentIndex); - float deltaFloat = partition.getFloat(channel, currentIndex); - return followFloat > currentFloat + deltaFloat; - case DOUBLE: - double currentDouble = column.getDouble(currentIndex); - double followDouble = column.getDouble(recentIndex); - double deltaDouble = partition.getDouble(channel, currentIndex); - return followDouble > currentDouble + deltaDouble; - default: - // Unreachable - throw new UnSupportedDataTypeException( - CalcMessages.UNSUPPORTED_DATA_TYPE + column.getDataType()); - } + return rangeFrameComparator.compare( + column, + partition, + currentIndex, + recentIndex, + channel, + TypeServices.RangeFrameOffsetOperation.ADD, + TypeServices.RangeFrameComparison.GREATER_THAN); } // Find first row which satisfy: @@ -377,34 +341,14 @@ private int getAscFrameStartPreceding(int currentIndex, int recentIndex) { private boolean compareInAscFrameStartPreceding(int currentIndex, int recentIndex, int channel) { checkArgument(!partition.isNull(channel, currentIndex)); - switch (column.getDataType()) { - case INT32: - case DATE: - int currentInt = column.getInt(currentIndex); - int precedeInt = column.getInt(recentIndex); - int deltaInt = partition.getInt(channel, currentIndex); - return precedeInt >= currentInt - deltaInt; - case INT64: - case TIMESTAMP: - long currentLong = column.getLong(currentIndex); - long precedeLong = column.getLong(recentIndex); - long deltaLong = partition.getLong(channel, currentIndex); - return precedeLong >= currentLong - deltaLong; - case FLOAT: - float currentFloat = column.getFloat(currentIndex); - float precedeFollow = column.getFloat(recentIndex); - float deltaFloat = partition.getFloat(channel, currentIndex); - return precedeFollow >= currentFloat - deltaFloat; - case DOUBLE: - double currentDouble = column.getDouble(currentIndex); - double precedeDouble = column.getDouble(recentIndex); - double deltaDouble = partition.getDouble(channel, currentIndex); - return precedeDouble >= currentDouble - deltaDouble; - default: - // Unreachable - throw new UnSupportedDataTypeException( - CalcMessages.UNSUPPORTED_DATA_TYPE + column.getDataType()); - } + return rangeFrameComparator.compare( + column, + partition, + currentIndex, + recentIndex, + channel, + TypeServices.RangeFrameOffsetOperation.SUBTRACT, + TypeServices.RangeFrameComparison.GREATER_THAN_OR_EQUAL); } // Find first row which satisfy: @@ -423,34 +367,14 @@ private int getAscFrameEndPreceding(int currentIndex, int recentIndex) { private boolean compareInAscFrameEndPreceding(int currentIndex, int recentIndex, int channel) { checkArgument(!partition.isNull(channel, currentIndex)); - switch (column.getDataType()) { - case INT32: - case DATE: - int currentInt = column.getInt(currentIndex); - int precedeInt = column.getInt(recentIndex); - int deltaInt = partition.getInt(channel, currentIndex); - return precedeInt > currentInt - deltaInt; - case INT64: - case TIMESTAMP: - long currentLong = column.getLong(currentIndex); - long precedeLong = column.getLong(recentIndex); - long deltaLong = partition.getLong(channel, currentIndex); - return precedeLong > currentLong - deltaLong; - case FLOAT: - float currentFloat = column.getFloat(currentIndex); - float precedeFollow = column.getFloat(recentIndex); - float deltaFloat = partition.getFloat(channel, currentIndex); - return precedeFollow > currentFloat - deltaFloat; - case DOUBLE: - double currentDouble = column.getDouble(currentIndex); - double precedeDouble = column.getDouble(recentIndex); - double deltaDouble = partition.getDouble(channel, currentIndex); - return precedeDouble > currentDouble - deltaDouble; - default: - // Unreachable - throw new UnSupportedDataTypeException( - CalcMessages.UNSUPPORTED_DATA_TYPE + column.getDataType()); - } + return rangeFrameComparator.compare( + column, + partition, + currentIndex, + recentIndex, + channel, + TypeServices.RangeFrameOffsetOperation.SUBTRACT, + TypeServices.RangeFrameComparison.GREATER_THAN); } // Find first row which satisfy: @@ -469,34 +393,14 @@ private int getDescFrameStartFollowing(int currentIndex, int recentIndex) { private boolean compareInDescFrameStartFollowing(int currentIndex, int recentIndex, int channel) { checkArgument(!partition.isNull(channel, currentIndex)); - switch (column.getDataType()) { - case INT32: - case DATE: - int currentInt = column.getInt(currentIndex); - int followInt = column.getInt(recentIndex); - int deltaInt = partition.getInt(channel, currentIndex); - return followInt <= currentInt - deltaInt; - case INT64: - case TIMESTAMP: - long currentLong = column.getLong(currentIndex); - long followLong = column.getLong(recentIndex); - long deltaLong = partition.getLong(channel, currentIndex); - return followLong <= currentLong - deltaLong; - case FLOAT: - float currentFloat = column.getFloat(currentIndex); - float followFloat = column.getFloat(recentIndex); - float deltaFloat = partition.getFloat(channel, currentIndex); - return followFloat <= currentFloat - deltaFloat; - case DOUBLE: - double currentDouble = column.getDouble(currentIndex); - double followDouble = column.getDouble(recentIndex); - double deltaDouble = partition.getDouble(channel, currentIndex); - return followDouble <= currentDouble - deltaDouble; - default: - // Unreachable - throw new UnSupportedDataTypeException( - CalcMessages.UNSUPPORTED_DATA_TYPE + column.getDataType()); - } + return rangeFrameComparator.compare( + column, + partition, + currentIndex, + recentIndex, + channel, + TypeServices.RangeFrameOffsetOperation.SUBTRACT, + TypeServices.RangeFrameComparison.LESS_THAN_OR_EQUAL); } // Find first row which satisfy: @@ -515,34 +419,14 @@ private int getDescFrameEndFollowing(int currentIndex, int recentIndex) { private boolean compareInDescFrameEndFollowing(int currentIndex, int recentIndex, int channel) { checkArgument(!partition.isNull(channel, currentIndex)); - switch (column.getDataType()) { - case INT32: - case DATE: - int currentInt = column.getInt(currentIndex); - int followInt = column.getInt(recentIndex); - int deltaInt = partition.getInt(channel, currentIndex); - return followInt < currentInt - deltaInt; - case INT64: - case TIMESTAMP: - long currentLong = column.getLong(currentIndex); - long followLong = column.getLong(recentIndex); - long deltaLong = partition.getLong(channel, currentIndex); - return followLong < currentLong - deltaLong; - case FLOAT: - float currentFloat = column.getFloat(currentIndex); - float followFloat = column.getFloat(recentIndex); - float deltaFloat = partition.getFloat(channel, currentIndex); - return followFloat < currentFloat - deltaFloat; - case DOUBLE: - double currentDouble = column.getDouble(currentIndex); - double followDouble = column.getDouble(recentIndex); - double deltaDouble = partition.getDouble(channel, currentIndex); - return followDouble < currentDouble - deltaDouble; - default: - // Unreachable - throw new UnSupportedDataTypeException( - CalcMessages.UNSUPPORTED_DATA_TYPE + column.getDataType()); - } + return rangeFrameComparator.compare( + column, + partition, + currentIndex, + recentIndex, + channel, + TypeServices.RangeFrameOffsetOperation.SUBTRACT, + TypeServices.RangeFrameComparison.LESS_THAN); } // Find first row which satisfy: @@ -561,34 +445,14 @@ private int getDescFrameStartPreceding(int currentIndex, int recentIndex) { private boolean compareInDescFrameStartPreceding(int currentIndex, int recentIndex, int channel) { checkArgument(!partition.isNull(channel, currentIndex)); - switch (column.getDataType()) { - case INT32: - case DATE: - int currentInt = column.getInt(currentIndex); - int precedeInt = column.getInt(recentIndex); - int deltaInt = partition.getInt(channel, currentIndex); - return precedeInt <= currentInt + deltaInt; - case INT64: - case TIMESTAMP: - long currentLong = column.getLong(currentIndex); - long precedeLong = column.getLong(recentIndex); - long deltaLong = partition.getLong(channel, currentIndex); - return precedeLong <= currentLong + deltaLong; - case FLOAT: - float currentFloat = column.getFloat(currentIndex); - float precedeFollow = column.getFloat(recentIndex); - float deltaFloat = partition.getFloat(channel, currentIndex); - return precedeFollow <= currentFloat + deltaFloat; - case DOUBLE: - double currentDouble = column.getDouble(currentIndex); - double precedeDouble = column.getDouble(recentIndex); - double deltaDouble = partition.getDouble(channel, currentIndex); - return precedeDouble <= currentDouble + deltaDouble; - default: - // Unreachable - throw new UnSupportedDataTypeException( - CalcMessages.UNSUPPORTED_DATA_TYPE + column.getDataType()); - } + return rangeFrameComparator.compare( + column, + partition, + currentIndex, + recentIndex, + channel, + TypeServices.RangeFrameOffsetOperation.ADD, + TypeServices.RangeFrameComparison.LESS_THAN_OR_EQUAL); } // Find first row which satisfy: @@ -607,33 +471,13 @@ private int getDescFrameEndPreceding(int currentIndex, int recentIndex) { private boolean compareInDescFrameEndPreceding(int currentIndex, int recentIndex, int channel) { checkArgument(!partition.isNull(channel, currentIndex)); - switch (column.getDataType()) { - case INT32: - case DATE: - int currentInt = column.getInt(currentIndex); - int precedeInt = column.getInt(recentIndex); - int deltaInt = partition.getInt(channel, currentIndex); - return precedeInt < currentInt + deltaInt; - case INT64: - case TIMESTAMP: - long currentLong = column.getLong(currentIndex); - long precedeLong = column.getLong(recentIndex); - long deltaLong = partition.getLong(channel, currentIndex); - return precedeLong < currentLong + deltaLong; - case FLOAT: - float currentFloat = column.getFloat(currentIndex); - float precedeFollow = column.getFloat(recentIndex); - float deltaFloat = partition.getFloat(channel, currentIndex); - return precedeFollow < currentFloat + deltaFloat; - case DOUBLE: - double currentDouble = column.getDouble(currentIndex); - double precedeDouble = column.getDouble(recentIndex); - double deltaDouble = partition.getDouble(channel, currentIndex); - return precedeDouble < currentDouble + deltaDouble; - default: - // Unreachable - throw new UnSupportedDataTypeException( - CalcMessages.UNSUPPORTED_DATA_TYPE + column.getDataType()); - } + return rangeFrameComparator.compare( + column, + partition, + currentIndex, + recentIndex, + channel, + TypeServices.RangeFrameOffsetOperation.ADD, + TypeServices.RangeFrameComparison.LESS_THAN); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/utils/ColumnList.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/utils/ColumnList.java index 03f2c20c5be9..0ea6b6878d54 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/utils/ColumnList.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/utils/ColumnList.java @@ -134,6 +134,15 @@ public Object getObject(int position) { return columns.get(columnIndex).getObject(offsetInColumn); } + public boolean arePositionsEqual(int position1, int position2) { + ColumnListIndex columnListIndex1 = getColumnListIndex(position1); + ColumnListIndex columnListIndex2 = getColumnListIndex(position2); + Column column1 = columns.get(columnListIndex1.getColumnIndex()); + Column column2 = columns.get(columnListIndex2.getColumnIndex()); + return column1.arePositionsEqual( + columnListIndex1.getOffsetInColumn(), column2, columnListIndex2.getOffsetInColumn()); + } + public boolean isNull(int position) { ColumnListIndex columnListIndex = getColumnListIndex(position); int columnIndex = columnListIndex.getColumnIndex(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/utils/RowComparator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/utils/RowComparator.java index 6f01fb96d94e..89388c257c91 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/utils/RowComparator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/window/utils/RowComparator.java @@ -21,8 +21,6 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.util.List; @@ -36,20 +34,14 @@ public RowComparator(List dataTypes) { public boolean equalColumns(List columns, int offset1, int offset2) { for (int i = 0; i < dataTypes.size(); i++) { Column column = columns.get(i); - TSDataType dataType = dataTypes.get(i); - if (!equal(column, dataType, offset1, offset2)) { + if (!equal(column, offset1, offset2)) { return false; } } return true; } - public boolean equal(Column column, int offset1, int offset2) { - assert dataTypes.size() == 1; - return equal(column, dataTypes.get(0), offset1, offset2); - } - - private boolean equal(Column column, TSDataType dataType, int offset1, int offset2) { + private boolean equal(Column column, int offset1, int offset2) { if (offset1 == offset2) { return true; } @@ -58,59 +50,7 @@ private boolean equal(Column column, TSDataType dataType, int offset1, int offse return column.isNull(offset1) && column.isNull(offset2); } - switch (dataType) { - case BOOLEAN: - boolean bool1 = column.getBoolean(offset1); - boolean bool2 = column.getBoolean(offset2); - if (bool1 != bool2) { - return false; - } - break; - case INT32: - case DATE: - int int1 = column.getInt(offset1); - int int2 = column.getInt(offset2); - if (int1 != int2) { - return false; - } - break; - case INT64: - case TIMESTAMP: - long long1 = column.getLong(offset1); - long long2 = column.getLong(offset2); - if (long1 != long2) { - return false; - } - break; - case FLOAT: - float float1 = column.getFloat(offset1); - float float2 = column.getFloat(offset2); - if (float1 != float2) { - return false; - } - break; - case DOUBLE: - double double1 = column.getDouble(offset1); - double double2 = column.getDouble(offset2); - if (double1 != double2) { - return false; - } - break; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - Binary bin1 = column.getBinary(offset1); - Binary bin2 = column.getBinary(offset2); - if (!bin1.equals(bin2)) { - return false; - } - break; - default: - // Would throw at the first run - throw new UnSupportedDataTypeException(dataType.toString()); - } - return true; + return column.arePositionsEqual(offset1, offset2); } public boolean equalColumnLists(List columns, int offset1, int offset2) { @@ -138,64 +78,11 @@ private boolean equal(ColumnList column, TSDataType dataType, int offset1, int o return column.isNull(offset1) && column.isNull(offset2); } - switch (dataType) { - case BOOLEAN: - boolean bool1 = column.getBoolean(offset1); - boolean bool2 = column.getBoolean(offset2); - if (bool1 != bool2) { - return false; - } - break; - case INT32: - case DATE: - int int1 = column.getInt(offset1); - int int2 = column.getInt(offset2); - if (int1 != int2) { - return false; - } - break; - case INT64: - case TIMESTAMP: - long long1 = column.getLong(offset1); - long long2 = column.getLong(offset2); - if (long1 != long2) { - return false; - } - break; - case FLOAT: - float float1 = column.getFloat(offset1); - float float2 = column.getFloat(offset2); - if (float1 != float2) { - return false; - } - break; - case DOUBLE: - double double1 = column.getDouble(offset1); - double double2 = column.getDouble(offset2); - if (double1 != double2) { - return false; - } - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - Binary bin1 = column.getBinary(offset1); - Binary bin2 = column.getBinary(offset2); - if (!bin1.equals(bin2)) { - return false; - } - break; - default: - // Would throw at the first run - throw new UnSupportedDataTypeException(dataType.toString()); - } - return true; + return column.arePositionsEqual(offset1, offset2); } public boolean equal(List columns1, int offset1, List columns2, int offset2) { for (int i = 0; i < dataTypes.size(); i++) { - TSDataType dataType = dataTypes.get(i); Column column1 = columns1.get(i); Column column2 = columns2.get(i); @@ -203,57 +90,8 @@ public boolean equal(List columns1, int offset1, List columns2, return column1.isNull(offset1) && column2.isNull(offset2); } - switch (dataType) { - case BOOLEAN: - boolean bool1 = column1.getBoolean(offset1); - boolean bool2 = column2.getBoolean(offset2); - if (bool1 != bool2) { - return false; - } - break; - case INT32: - case DATE: - int int1 = column1.getInt(offset1); - int int2 = column2.getInt(offset2); - if (int1 != int2) { - return false; - } - break; - case INT64: - case TIMESTAMP: - long long1 = column1.getLong(offset1); - long long2 = column2.getLong(offset2); - if (long1 != long2) { - return false; - } - break; - case FLOAT: - float float1 = column1.getFloat(offset1); - float float2 = column2.getFloat(offset2); - if (float1 != float2) { - return false; - } - break; - case DOUBLE: - double double1 = column1.getDouble(offset1); - double double2 = column2.getDouble(offset2); - if (double1 != double2) { - return false; - } - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - Binary bin1 = column1.getBinary(offset1); - Binary bin2 = column2.getBinary(offset2); - if (!bin1.equals(bin2)) { - return false; - } - break; - default: - // Would throw at the first run - throw new UnSupportedDataTypeException(dataType.toString()); + if (!column1.arePositionsEqual(offset1, column2, offset2)) { + return false; } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/MergeSortFullOuterJoinOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/MergeSortFullOuterJoinOperator.java index 278d87ea0215..cfb2c587a40a 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/MergeSortFullOuterJoinOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/MergeSortFullOuterJoinOperator.java @@ -23,6 +23,7 @@ import org.apache.iotdb.calc.execution.operator.Operator; import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.JoinKeyComparator; import org.apache.iotdb.calc.plan.planner.CommonOperatorUtils; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.queryengine.execution.MemoryEstimationHelper; import org.apache.tsfile.block.column.Column; @@ -32,7 +33,6 @@ import org.apache.tsfile.utils.RamUsageEstimator; import java.util.List; -import java.util.function.BiFunction; public class MergeSortFullOuterJoinOperator extends AbstractMergeSortJoinOperator { private static final long INSTANCE_SIZE = @@ -41,7 +41,7 @@ public class MergeSortFullOuterJoinOperator extends AbstractMergeSortJoinOperato // stores last row matched join criteria, only used in outer join private TsBlock lastMatchedRightBlock = null; private final int[] lastMatchedBlockPositions; - private final List> updateLastMatchedRowFunctions; + private final List updateLastMatchedRowFunctions; public MergeSortFullOuterJoinOperator( CommonOperatorContext operatorContext, @@ -53,7 +53,7 @@ public MergeSortFullOuterJoinOperator( int[] rightOutputSymbolIdx, List joinKeyComparators, List dataTypes, - List> updateLastMatchedRowFunctions) { + List updateLastMatchedRowFunctions) { super( operatorContext, leftChild, diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxMostFrequentAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxMostFrequentAccumulator.java index c5dcb5837129..0b2d9ab5f0ad 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxMostFrequentAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxMostFrequentAccumulator.java @@ -53,7 +53,7 @@ public boolean hasFinalResult() { @Override public void addStatistics(Statistics[] statistics) { throw new UnsupportedOperationException( - CalcMessages.EXCEPTION_APPROXMOSTFREQUENTACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS_DB1E218B); + CalcMessages.APPROX_MOST_FREQUENT_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxPercentileAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxPercentileAccumulator.java index 73cf3e66badc..69804e03b64e 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxPercentileAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxPercentileAccumulator.java @@ -21,18 +21,20 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.TDigest; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; +import java.util.function.BiConsumer; public abstract class AbstractApproxPercentileAccumulator implements TableAccumulator { private static final long INSTANCE_SIZE = @@ -40,10 +42,18 @@ public abstract class AbstractApproxPercentileAccumulator implements TableAccumu protected final TDigest tDigest = new TDigest(); protected final TSDataType seriesDataType; + private final TypeServices.NumericResultWriter resultWriter; + private final BiConsumer inputAdder; protected double percentage; AbstractApproxPercentileAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; + this.resultWriter = + TypeServices.NUMERIC_RESULT_WRITER_SERVICE.call(Type.fromTsDataType(seriesDataType)); + this.inputAdder = + TypeServices.APPROX_PERCENTILE_INPUT_SERVICE + .call(Type.fromTsDataType(seriesDataType)) + .apply(this); } @Override @@ -65,30 +75,9 @@ public void addInput(Column[] arguments, AggregationMask mask) { } else { throw new SemanticException( String.format( - CalcMessages.EXCEPTION_APPROX_PERCENTILE_REQUIRES_2_3_ARGUMENTS_BUT_GOT_ARG_D78590AA, - arguments.length)); - } - switch (seriesDataType) { - case INT32: - addIntInput(arguments, mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(arguments, mask); - return; - case FLOAT: - addFloatInput(arguments, mask); - return; - case DOUBLE: - addDoubleInput(arguments, mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages - .EXCEPTION_UNSUPPORTED_DATA_TYPE_APPROX_PERCENTILE_AGGREGATION_ARG_CFEC0431, - seriesDataType)); + CalcMessages.APPROX_PERCENTILE_REQUIRES_TWO_OR_THREE_ARGUMENTS, arguments.length)); } + inputAdder.accept(arguments, mask); } @Override @@ -122,27 +111,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (seriesDataType) { - case INT32: - columnBuilder.writeInt((int) result); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong((long) result); - break; - case FLOAT: - columnBuilder.writeFloat((float) result); - break; - case DOUBLE: - columnBuilder.writeDouble(result); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages - .EXCEPTION_UNSUPPORTED_DATA_TYPE_APPROX_PERCENTILE_AGGREGATION_ARG_CFEC0431, - seriesDataType)); - } + resultWriter.write(columnBuilder, result); } @Override @@ -153,7 +122,7 @@ public boolean hasFinalResult() { @Override public void addStatistics(Statistics[] statistics) { throw new UnsupportedOperationException( - CalcMessages.EXCEPTION_APPROXPERCENTILEACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS_2BB01365); + CalcMessages.APPROX_PERCENTILE_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS); } @Override @@ -173,8 +142,7 @@ public static double toDoubleExact(long value) { double doubleValue = (double) value; if ((long) doubleValue != value) { throw new SemanticException( - String.format( - CalcMessages.EXCEPTION_NO_EXACT_DOUBLE_REPRESENTATION_LONG_ARG_3E5B7550, value)); + String.format(CalcMessages.NO_EXACT_DOUBLE_REPRESENTATION_FOR_LONG, value)); } return value; } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AccumulatorFactory.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AccumulatorFactory.java index 8e90bd8e6aed..e1743f617130 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AccumulatorFactory.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AccumulatorFactory.java @@ -23,11 +23,6 @@ import org.apache.iotdb.calc.execution.aggregation.CovarianceAccumulator; import org.apache.iotdb.calc.execution.aggregation.RegressionAccumulator; import org.apache.iotdb.calc.execution.aggregation.VarianceAccumulator; -import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.BinaryGroupedApproxMostFrequentAccumulator; -import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.BlobGroupedApproxMostFrequentAccumulator; -import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.BooleanGroupedApproxMostFrequentAccumulator; -import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.DoubleGroupedApproxMostFrequentAccumulator; -import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.FloatGroupedApproxMostFrequentAccumulator; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedAccumulator; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedApproxCountDistinctAccumulator; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedApproxPercentileAccumulator; @@ -54,8 +49,6 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedSumAccumulator; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedUserDefinedAggregateAccumulator; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedVarianceAccumulator; -import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.IntGroupedApproxMostFrequentAccumulator; -import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.LongGroupedApproxMostFrequentAccumulator; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate.GroupedNaiveDeltaAccumulator; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate.GroupedNaiveIncreaseAccumulator; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate.GroupedNaiveIrateAccumulator; @@ -74,6 +67,7 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.OrderedRateAccumulator; import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.common.rpc.thrift.TAggregationType; import org.apache.iotdb.commons.queryengine.execution.operator.source.relational.aggregation.grouped.UpdateMemory; import org.apache.iotdb.commons.queryengine.execution.operator.source.relational.aggregation.grouped.hash.MarkDistinctHash; @@ -94,7 +88,6 @@ import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeFactory; -import org.apache.tsfile.write.UnSupportedDataTypeException; import javax.annotation.Nullable; @@ -641,61 +634,12 @@ public static TableAccumulator createBuiltinAccumulator( } public static GroupedAccumulator getGroupedApproxMostFrequentAccumulator(TSDataType type) { - switch (type) { - case BOOLEAN: - return new BooleanGroupedApproxMostFrequentAccumulator(); - case INT32: - case DATE: - return new IntGroupedApproxMostFrequentAccumulator(); - case INT64: - case TIMESTAMP: - return new LongGroupedApproxMostFrequentAccumulator(); - case FLOAT: - return new FloatGroupedApproxMostFrequentAccumulator(); - case DOUBLE: - return new DoubleGroupedApproxMostFrequentAccumulator(); - case TEXT: - case STRING: - return new BinaryGroupedApproxMostFrequentAccumulator(); - case BLOB: - return new BlobGroupedApproxMostFrequentAccumulator(); - case OBJECT: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages - .EXCEPTION_UNSUPPORTED_DATA_TYPE_APPROX_COUNT_DISTINCT_AGGREGATION_ARG_58F0391E, - type)); - } + return TypeServices.GROUPED_APPROX_MOST_FREQUENT_ACCUMULATOR_SERVICE.call( + Type.fromTsDataType(type)); } public static TableAccumulator getApproxMostFrequentAccumulator(TSDataType type) { - switch (type) { - case BOOLEAN: - return new BooleanApproxMostFrequentAccumulator(); - case INT32: - case DATE: - return new IntApproxMostFrequentAccumulator(); - case INT64: - case TIMESTAMP: - return new LongApproxMostFrequentAccumulator(); - case FLOAT: - return new FloatApproxMostFrequentAccumulator(); - case DOUBLE: - return new DoubleApproxMostFrequentAccumulator(); - case TEXT: - case STRING: - return new BinaryApproxMostFrequentAccumulator(); - case BLOB: - return new BlobApproxMostFrequentAccumulator(); - case OBJECT: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages - .EXCEPTION_UNSUPPORTED_DATA_TYPE_APPROX_COUNT_DISTINCT_AGGREGATION_ARG_58F0391E, - type)); - } + return TypeServices.APPROX_MOST_FREQUENT_ACCUMULATOR_SERVICE.call(Type.fromTsDataType(type)); } public static boolean isMultiInputAggregation(TAggregationType aggregationType) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java index 4c5f15e4e04f..65824142a079 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java @@ -22,15 +22,16 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.HyperLogLog; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.HyperLogLogStateFactory; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.write.UnSupportedDataTypeException; import static com.google.common.base.Preconditions.checkArgument; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.HyperLogLog.DEFAULT_STANDARD_ERROR; @@ -39,6 +40,7 @@ public class ApproxCountDistinctAccumulator implements TableAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ApproxCountDistinctAccumulator.class); private final TSDataType seriesDataType; + private final TypeServices.HyperLogLogColumnAdder valueAdder; private final HyperLogLogStateFactory.SingleHyperLogLogState state = HyperLogLogStateFactory.createSingleState(); @@ -46,6 +48,8 @@ public class ApproxCountDistinctAccumulator implements TableAccumulator { public ApproxCountDistinctAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; + this.valueAdder = + TypeServices.HYPER_LOG_LOG_COLUMN_ADDER_SERVICE.call(Type.fromTsDataType(seriesDataType)); } @Override @@ -66,36 +70,22 @@ public void addInput(Column[] arguments, AggregationMask mask) { arguments.length == 1 ? DEFAULT_STANDARD_ERROR : arguments[1].getDouble(0); HyperLogLog hll = getOrCreateHyperLogLog(state, maxStandardError); - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(arguments[0], mask, hll); - return; - case INT64: - case TIMESTAMP: - addLongInput(arguments[0], mask, hll); - return; - case FLOAT: - addFloatInput(arguments[0], mask, hll); - return; - case DOUBLE: - addDoubleInput(arguments[0], mask, hll); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(arguments[0], mask, hll); - return; - case BOOLEAN: - addBooleanInput(arguments[0], mask, hll); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages - .EXCEPTION_UNSUPPORTED_DATA_TYPE_APPROX_COUNT_DISTINCT_AGGREGATION_ARG_58F0391E, - seriesDataType)); + Column valueColumn = arguments[0]; + int positionCount = mask.getSelectedPositionCount(); + if (mask.isSelectAll()) { + for (int i = 0; i < positionCount; i++) { + if (!valueColumn.isNull(i)) { + valueAdder.add(hll, valueColumn, i); + } + } + } else { + int[] selectedPositions = mask.getSelectedPositions(); + for (int i = 0; i < positionCount; i++) { + int position = selectedPositions[i]; + if (!valueColumn.isNull(position)) { + valueAdder.add(hll, valueColumn, position); + } + } } } @@ -132,7 +122,7 @@ public boolean hasFinalResult() { @Override public void addStatistics(Statistics[] statistics) { throw new UnsupportedOperationException( - CalcMessages.EXCEPTION_APPROXCOUNTDISTINCTACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS_81005A79); + CalcMessages.APPROX_COUNT_DISTINCT_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AvgAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AvgAccumulator.java index 0c4834569b30..62efaf58fab6 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AvgAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AvgAccumulator.java @@ -19,6 +19,7 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -28,6 +29,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -39,12 +41,23 @@ public class AvgAccumulator implements TableAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(AvgAccumulator.class); private final TSDataType argumentDataType; + private final TypeServices.ColumnToDoubleConverter valueConverter; private long countValue; private double sumValue; private boolean initResult = false; public AvgAccumulator(TSDataType argumentDataType) { this.argumentDataType = argumentDataType; + Type type = Type.fromTsDataType(argumentDataType); + this.valueConverter = + TypeServices.AGGREGATION_NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(type) + .create( + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_AGGREGATION_AVG, + argumentDataType))); } @Override @@ -59,68 +72,14 @@ public TableAccumulator copy() { @Override public void addInput(Column[] arguments, AggregationMask mask) { - checkArgument( - arguments.length == 1, - CalcMessages.EXCEPTION_ARGUMENT_OF_AVG_SHOULD_BE_ONE_COLUMN_82162B82); - switch (argumentDataType) { - case INT32: - addIntInput(arguments[0], mask); - return; - case INT64: - addLongInput(arguments[0], mask); - return; - case FLOAT: - addFloatInput(arguments[0], mask); - return; - case DOUBLE: - addDoubleInput(arguments[0], mask); - return; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_AGGREGATION_AVG_ARG_4E63A3C3, - argumentDataType)); - } + checkArgument(arguments.length == 1, "argument of AVG should be one column"); + addInput(arguments[0], mask); } @Override public void removeInput(Column[] arguments) { - checkArgument( - arguments.length == 1, - CalcMessages.EXCEPTION_ARGUMENT_OF_AVG_SHOULD_BE_ONE_COLUMN_82162B82); - switch (argumentDataType) { - case INT32: - removeIntInput(arguments[0]); - return; - case INT64: - removeLongInput(arguments[0]); - return; - case FLOAT: - removeFloatInput(arguments[0]); - return; - case DOUBLE: - removeDoubleInput(arguments[0]); - return; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_AGGREGATION_AVG_ARG_4E63A3C3, - argumentDataType)); - } + checkArgument(arguments.length == 1, "argument of AVG should be one column"); + removeInput(arguments[0]); } @Override @@ -208,82 +167,7 @@ private byte[] serializeState() { return bytes; } - private void addIntInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - initResult = true; - countValue++; - sumValue += column.getInt(i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - initResult = true; - countValue++; - sumValue += column.getInt(position); - } - } - } - } - - private void addLongInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - initResult = true; - countValue++; - sumValue += column.getLong(i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - initResult = true; - countValue++; - sumValue += column.getLong(position); - } - } - } - } - - private void addFloatInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - initResult = true; - countValue++; - sumValue += column.getFloat(i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - initResult = true; - countValue++; - sumValue += column.getFloat(position); - } - } - } - } - - private void addDoubleInput(Column column, AggregationMask mask) { + private void addInput(Column column, AggregationMask mask) { int positionCount = mask.getSelectedPositionCount(); if (mask.isSelectAll()) { @@ -291,7 +175,7 @@ private void addDoubleInput(Column column, AggregationMask mask) { if (!column.isNull(i)) { initResult = true; countValue++; - sumValue += column.getDouble(i); + sumValue += valueConverter.convert(column, i); } } } else { @@ -302,48 +186,18 @@ private void addDoubleInput(Column column, AggregationMask mask) { if (!column.isNull(position)) { initResult = true; countValue++; - sumValue += column.getDouble(position); + sumValue += valueConverter.convert(column, position); } } } } - private void removeIntInput(Column column) { - int count = column.getPositionCount(); - for (int i = 0; i < count; i++) { - if (!column.isNull(i)) { - countValue--; - sumValue -= column.getInt(i); - } - } - } - - private void removeLongInput(Column column) { - int count = column.getPositionCount(); - for (int i = 0; i < count; i++) { - if (!column.isNull(i)) { - countValue--; - sumValue -= column.getLong(i); - } - } - } - - private void removeFloatInput(Column column) { - int count = column.getPositionCount(); - for (int i = 0; i < count; i++) { - if (!column.isNull(i)) { - countValue--; - sumValue -= column.getFloat(i); - } - } - } - - private void removeDoubleInput(Column column) { + private void removeInput(Column column) { int count = column.getPositionCount(); for (int i = 0; i < count; i++) { if (!column.isNull(i)) { countValue--; - sumValue -= column.getDouble(i); + sumValue -= valueConverter.convert(column, i); } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ExtremeAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ExtremeAccumulator.java index f546282d31b6..cbcdecf94f1f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ExtremeAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ExtremeAccumulator.java @@ -19,25 +19,36 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; + import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.util.function.Supplier; + public class ExtremeAccumulator implements TableAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ExtremeAccumulator.class); - private static final String UNSUPPORTED_DATA_TYPE = "Unsupported data type in EXTREME: %s"; private final TSDataType seriesDataType; + private final Type type; private final TsPrimitiveType extremeResult; + private final TypeServices.ColumnValueUpdater valueUpdater; + private final TypeServices.StatisticsValueUpdater statisticsValueUpdater; private boolean initResult; public ExtremeAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - this.extremeResult = TsPrimitiveType.getByType(seriesDataType); + this.type = Type.fromTsDataType(seriesDataType); + this.extremeResult = type.getTsPrimitiveType(); + this.valueUpdater = TypeServices.EXTREME_COLUMN_VALUE_UPDATER_SERVICE.call(type); + this.statisticsValueUpdater = TypeServices.EXTREME_STATISTICS_VALUE_UPDATER_SERVICE.call(type); } @Override @@ -52,29 +63,22 @@ public TableAccumulator copy() { @Override public void addInput(Column[] arguments, AggregationMask mask) { - switch (seriesDataType) { - case INT32: - addIntInput(arguments[0], mask); - return; - case INT64: - addLongInput(arguments[0], mask); - return; - case FLOAT: - addFloatInput(arguments[0], mask); - return; - case DOUBLE: - addDoubleInput(arguments[0], mask); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format(UNSUPPORTED_DATA_TYPE, seriesDataType)); + Column column = arguments[0]; + int positionCount = mask.getSelectedPositionCount(); + if (mask.isSelectAll()) { + for (int i = 0; i < positionCount; i++) { + if (!column.isNull(i)) { + initResult |= valueUpdater.update(extremeResult, column, i, initResult); + } + } + } else { + int[] selectedPositions = mask.getSelectedPositions(); + for (int i = 0; i < positionCount; i++) { + int position = selectedPositions[i]; + if (!column.isNull(position)) { + initResult |= valueUpdater.update(extremeResult, column, position, initResult); + } + } } } @@ -85,30 +89,7 @@ public void addIntermediate(Column argument) { continue; } - switch (seriesDataType) { - case INT32: - updateIntResult(argument.getInt(i)); - break; - case INT64: - updateLongResult(argument.getLong(i)); - break; - case FLOAT: - updateFloatResult(argument.getFloat(i)); - break; - case DOUBLE: - updateDoubleResult(argument.getDouble(i)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format(UNSUPPORTED_DATA_TYPE, seriesDataType)); - } + initResult |= valueUpdater.update(extremeResult, argument, i, initResult); } } @@ -118,33 +99,18 @@ public void addStatistics(Statistics[] statistics) { return; } - switch (seriesDataType) { - case INT32: - updateIntResult(((Number) statistics[0].getMaxValue()).intValue()); - updateIntResult(((Number) statistics[0].getMinValue()).intValue()); - break; - case INT64: - updateLongResult(((Number) statistics[0].getMaxValue()).longValue()); - updateLongResult(((Number) statistics[0].getMinValue()).longValue()); - break; - case FLOAT: - updateFloatResult(((Number) statistics[0].getMaxValue()).floatValue()); - updateFloatResult(((Number) statistics[0].getMinValue()).floatValue()); - break; - case DOUBLE: - updateDoubleResult(((Number) statistics[0].getMaxValue()).doubleValue()); - updateDoubleResult(((Number) statistics[0].getMinValue()).doubleValue()); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format(UNSUPPORTED_DATA_TYPE, seriesDataType)); + Supplier exceptionSupplier = + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_EXTREME_AGGREGATION, seriesDataType)); + if (statisticsValueUpdater.update( + extremeResult, statistics[0].getMaxValue(), initResult, exceptionSupplier)) { + initResult = true; + } + if (statisticsValueUpdater.update( + extremeResult, statistics[0].getMinValue(), initResult, exceptionSupplier)) { + initResult = true; } } @@ -155,30 +121,7 @@ public void evaluateIntermediate(ColumnBuilder columnBuilder) { return; } - switch (seriesDataType) { - case INT32: - columnBuilder.writeInt(extremeResult.getInt()); - break; - case INT64: - columnBuilder.writeLong(extremeResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(extremeResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(extremeResult.getDouble()); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format(UNSUPPORTED_DATA_TYPE, seriesDataType)); - } + type.write(columnBuilder, extremeResult); } @Override @@ -188,30 +131,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { return; } - switch (seriesDataType) { - case INT32: - columnBuilder.writeInt(extremeResult.getInt()); - break; - case INT64: - columnBuilder.writeLong(extremeResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(extremeResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(extremeResult.getDouble()); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format(UNSUPPORTED_DATA_TYPE, seriesDataType)); - } + type.write(columnBuilder, extremeResult); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/FirstAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/FirstAccumulator.java index fe2faa9ca57a..a447a9a291d4 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/FirstAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/FirstAccumulator.java @@ -20,6 +20,7 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -29,12 +30,13 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import static com.google.common.base.Preconditions.checkArgument; @@ -45,6 +47,9 @@ public class FirstAccumulator implements TableAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(FirstAccumulator.class); protected final TSDataType seriesDataType; + protected final Type type; + private final TypeServices.PrimitiveColumnValueSetter valueSetter; + private final TypeServices.StatisticsValueSetter statisticsValueSetter; protected TsPrimitiveType firstValue; protected long minTime = Long.MAX_VALUE; protected boolean initResult = false; @@ -53,7 +58,10 @@ public class FirstAccumulator implements TableAccumulator { public FirstAccumulator(TSDataType seriesDataType, boolean canFinishAfterInit) { this.seriesDataType = seriesDataType; - this.firstValue = TsPrimitiveType.getByType(seriesDataType); + this.type = Type.fromTsDataType(seriesDataType); + this.firstValue = type.getTsPrimitiveType(); + this.valueSetter = TypeServices.PRIMITIVE_COLUMN_VALUE_SETTER_SERVICE.call(type); + this.statisticsValueSetter = TypeServices.STATISTICS_VALUE_SETTER_SERVICE.call(type); this.canFinishAfterInit = canFinishAfterInit; } @@ -70,35 +78,7 @@ public TableAccumulator copy() { @Override public void addInput(Column[] arguments, AggregationMask mask) { // arguments[0] is value column, arguments[1] is time column - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(arguments[0], arguments[1], mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(arguments[0], arguments[1], mask); - return; - case FLOAT: - addFloatInput(arguments[0], arguments[1], mask); - return; - case DOUBLE: - addDoubleInput(arguments[0], arguments[1], mask); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(arguments[0], arguments[1], mask); - return; - case BOOLEAN: - addBooleanInput(arguments[0], arguments[1], mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_ARG_D5B6C6F9, seriesDataType)); - } + addInput(arguments[0], arguments[1], mask); } @Override @@ -115,72 +95,14 @@ public void addIntermediate(Column argument) { continue; } - byte[] bytes = argument.getBinary(i).getValues(); - long time = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); - boolean isTimeNull = BytesUtils.bytesToBool(bytes, 8); - int offset = 9; - - switch (seriesDataType) { - case INT32: - case DATE: - int intVal = BytesUtils.bytesToInt(bytes, offset); - if (!isTimeNull) { - updateIntFirstValue(intVal, time); - } else { - updateIntNullTimeValue(intVal); - } - break; - case INT64: - case TIMESTAMP: - long longVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - if (!isTimeNull) { - updateLongFirstValue(longVal, time); - } else { - updateLongNullTimeValue(longVal); - } - break; - case FLOAT: - float floatVal = BytesUtils.bytesToFloat(bytes, offset); - if (!isTimeNull) { - updateFloatFirstValue(floatVal, time); - } else { - updateFloatNullTimeValue(floatVal); - } - break; - case DOUBLE: - double doubleVal = BytesUtils.bytesToDouble(bytes, offset); - if (!isTimeNull) { - updateDoubleFirstValue(doubleVal, time); - } else { - updateDoubleNullTimeValue(doubleVal); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - if (!isTimeNull) { - updateBinaryFirstValue(binaryVal, time); - } else { - updateBinaryNullTimeValue(binaryVal); - } - break; - case BOOLEAN: - boolean boolVal = BytesUtils.bytesToBool(bytes, offset); - if (!isTimeNull) { - updateBooleanFirstValue(boolVal, time); - } else { - updateBooleanNullTimeValue(boolVal); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_B2D27BB9, - seriesDataType)); + ByteBuffer buffer = ByteBuffer.wrap(argument.getBinary(i).getValues()); + long time = buffer.getLong(); + boolean isTimeNull = buffer.get() != 0; + TsPrimitiveType value = type.deserialize(buffer); + if (!isTimeNull) { + updateFirstValue(value, time); + } else { + updateNullTimeValue(value); } } } @@ -214,36 +136,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(firstValue.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(firstValue.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(firstValue.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(firstValue.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - columnBuilder.writeBinary(firstValue.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(firstValue.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_2652C363, - seriesDataType)); - } + type.write(columnBuilder, firstValue); } @Override @@ -251,60 +144,51 @@ public boolean hasFinalResult() { return canFinishAfterInit && initResult; } + private void addInput(Column valueColumn, Column timeColumn, AggregationMask mask) { + int selectPositionCount = mask.getSelectedPositionCount(); + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; + } + + if (!timeColumn.isNull(position)) { + if (checkAndUpdateFirstTime(timeColumn.getLong(position))) { + valueSetter.set(firstValue, valueColumn, position); + } + if (canFinishAfterInit) { + return; + } + } else if (checkAndUpdateNullTime()) { + valueSetter.set(firstValue, valueColumn, position); + } + } + } + @Override public void addStatistics(Statistics[] statistics) { if (statistics == null || statistics[0] == null) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntFirstValue( - ((Number) statistics[0].getFirstValue()).intValue(), statistics[0].getStartTime()); - break; - case INT64: - case TIMESTAMP: - updateLongFirstValue( - ((Number) statistics[0].getFirstValue()).longValue(), statistics[0].getStartTime()); - break; - case FLOAT: - updateFloatFirstValue( - ((Number) statistics[0].getFirstValue()).floatValue(), statistics[0].getStartTime()); - break; - case DOUBLE: - updateDoubleFirstValue( - ((Number) statistics[0].getFirstValue()).doubleValue(), statistics[0].getStartTime()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - if (statistics[0] instanceof DateStatistics) { - updateBinaryFirstValue( - new Binary( - TSDataType.getDateStringValue((Integer) statistics[0].getFirstValue()), - StandardCharsets.UTF_8), - statistics[0].getStartTime()); - } else { - if (statistics[0].getFirstValue() instanceof Binary) { - updateBinaryFirstValue( - (Binary) statistics[0].getFirstValue(), statistics[0].getStartTime()); - } else { - updateBinaryFirstValue( - new Binary(String.valueOf(statistics[0].getFirstValue()), StandardCharsets.UTF_8), - statistics[0].getStartTime()); - } - } - break; - case BOOLEAN: - updateBooleanFirstValue( - (boolean) statistics[0].getFirstValue(), statistics[0].getStartTime()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_B2D27BB9, - seriesDataType)); + Object value = statistics[0].getFirstValue(); + if (statistics[0] instanceof DateStatistics + && (seriesDataType == TSDataType.TEXT + || seriesDataType == TSDataType.STRING + || seriesDataType == TSDataType.BLOB + || seriesDataType == TSDataType.OBJECT)) { + value = new Binary(TSDataType.getDateStringValue((Integer) value), StandardCharsets.UTF_8); + } + if (checkAndUpdateFirstTime(statistics[0].getStartTime())) { + statisticsValueSetter.set( + firstValue, + value, + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_FIRST_AGGREGATION, seriesDataType))); } } @@ -333,6 +217,18 @@ private boolean checkAndUpdateNullTime() { return false; } + private void updateFirstValue(TsPrimitiveType value, long curTime) { + if (checkAndUpdateFirstTime(curTime)) { + firstValue.copy(value); + } + } + + private void updateNullTimeValue(TsPrimitiveType value) { + if (checkAndUpdateNullTime()) { + firstValue.copy(value); + } + } + /** * Updates the accumulator, prioritizing values with valid timestamps over those with null * timestamps. diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/FirstByAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/FirstByAccumulator.java index 22b6da926bea..6ed27aaaa8b5 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/FirstByAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/FirstByAccumulator.java @@ -20,6 +20,7 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -28,13 +29,14 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; -import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; import static com.google.common.base.Preconditions.checkArgument; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.Utils.serializeTimeValueWithNull; @@ -52,6 +54,9 @@ public class FirstByAccumulator implements TableAccumulator { private long yFirstTime = Long.MAX_VALUE; + private final Type xType; + private final TypeServices.PrimitiveColumnValueSetter xValueSetter; + private final TypeServices.StatisticsValueSetter xStatisticsValueSetter; private final TsPrimitiveType xResult; private boolean xIsNull = true; @@ -71,7 +76,10 @@ public FirstByAccumulator( this.xIsTimeColumn = xIsTimeColumn; this.yIsTimeColumn = yIsTimeColumn; - this.xResult = TsPrimitiveType.getByType(xDataType); + this.xType = Type.fromTsDataType(xDataType); + this.xValueSetter = TypeServices.PRIMITIVE_COLUMN_VALUE_SETTER_SERVICE.call(xType); + this.xStatisticsValueSetter = TypeServices.STATISTICS_VALUE_SETTER_SERVICE.call(xType); + this.xResult = xType.getTsPrimitiveType(); this.canFinishAfterInit = canFinishAfterInit; } @@ -94,36 +102,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { .EXCEPTION_LENGTH_OF_INPUT_COLUMN_LEFT_BRACKET_RIGHT_BRACKET_FOR_FIRST_BY_SHOULD_BE_3_BE623E5E); // arguments[0] is x column, arguments[1] is y column, arguments[2] is time column - switch (xDataType) { - case INT32: - case DATE: - addIntInput(arguments[0], arguments[1], arguments[2], mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(arguments[0], arguments[1], arguments[2], mask); - return; - case FLOAT: - addFloatInput(arguments[0], arguments[1], arguments[2], mask); - return; - case DOUBLE: - addDoubleInput(arguments[0], arguments[1], arguments[2], mask); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(arguments[0], arguments[1], arguments[2], mask); - return; - case BOOLEAN: - addBooleanInput(arguments[0], arguments[1], arguments[2], mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - yDataType)); - } + addInput(arguments[0], arguments[1], arguments[2], mask); } @Override @@ -148,80 +127,16 @@ public void addIntermediate(Column argument) { boolean isXValueNull = BytesUtils.bytesToBool(bytes, offset); offset += 1; - switch (xDataType) { - case INT32: - case DATE: - int intVal = isXValueNull ? 0 : BytesUtils.bytesToInt(bytes, offset); - if (!isOrderTimeNull) { - updateIntFirstValue(isXValueNull, intVal, curTime); - } else { - updateIntNullTimeValue(isXValueNull, intVal); - } - break; - - case INT64: - case TIMESTAMP: - long longVal = - isXValueNull ? 0 : BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - if (!isOrderTimeNull) { - updateLongFirstValue(isXValueNull, longVal, curTime); - } else { - updateLongNullTimeValue(isXValueNull, longVal); - } - break; - - case FLOAT: - float floatVal = isXValueNull ? 0 : BytesUtils.bytesToFloat(bytes, offset); - if (!isOrderTimeNull) { - updateFloatFirstValue(isXValueNull, floatVal, curTime); - } else { - updateFloatNullTimeValue(isXValueNull, floatVal); - } - break; - - case DOUBLE: - double doubleVal = isXValueNull ? 0 : BytesUtils.bytesToDouble(bytes, offset); - if (!isOrderTimeNull) { - updateDoubleFirstValue(isXValueNull, doubleVal, curTime); - } else { - updateDoubleNullTimeValue(isXValueNull, doubleVal); - } - break; - - case TEXT: - case BLOB: - case OBJECT: - case STRING: - Binary binaryVal = null; - if (!isXValueNull) { - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - } - if (!isOrderTimeNull) { - updateBinaryFirstValue(isXValueNull, binaryVal, curTime); - } else { - updateBinaryNullTimeValue(isXValueNull, binaryVal); - } - break; - - case BOOLEAN: - boolean boolVal = false; - if (!isXValueNull) { - boolVal = BytesUtils.bytesToBool(bytes, offset); - } - if (!isOrderTimeNull) { - updateBooleanFirstValue(isXValueNull, boolVal, curTime); - } else { - updateBooleanNullTimeValue(isXValueNull, boolVal); - } - break; - - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - yDataType)); + TsPrimitiveType value = + isXValueNull + ? null + : xType.deserialize(ByteBuffer.wrap(bytes, offset, bytes.length - offset)); + if (!isOrderTimeNull) { + if (checkAndUpdateFirstTime(isXValueNull, curTime) && !isXValueNull) { + xResult.copy(value); + } + } else if (checkAndUpdateNullTime(isXValueNull) && !isXValueNull) { + xResult.copy(value); } } } @@ -253,36 +168,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { return; } - switch (xDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(xResult.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(xResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(xResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(xResult.getDouble()); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - columnBuilder.writeBinary(xResult.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(xResult.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - xDataType)); - } + xType.write(columnBuilder, xResult); } @Override @@ -315,42 +201,13 @@ public void addStatistics(Statistics[] statistics) { yFirstTime = yStatistics.getStartTime(); xIsNull = false; - switch (xDataType) { - case INT32: - case DATE: - xResult.setInt(((Number) xStatistics.getFirstValue()).intValue()); - break; - case INT64: - case TIMESTAMP: - xResult.setLong(((Number) xStatistics.getFirstValue()).longValue()); - break; - case FLOAT: - xResult.setFloat(((Number) statistics[0].getFirstValue()).floatValue()); - break; - case DOUBLE: - xResult.setDouble(((Number) statistics[0].getFirstValue()).doubleValue()); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - if (statistics[0].getFirstValue() instanceof Binary) { - xResult.setBinary((Binary) statistics[0].getFirstValue()); - } else { - xResult.setBinary( - new Binary( - String.valueOf(statistics[0].getFirstValue()), StandardCharsets.UTF_8)); - } - break; - case BOOLEAN: - xResult.setBoolean((boolean) statistics[0].getFirstValue()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - yDataType)); - } + xStatisticsValueSetter.set( + xResult, + xStatistics.getFirstValue(), + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_FIRST_BY_AGGREGATION, yDataType))); } } } else { @@ -404,6 +261,31 @@ private boolean checkAndUpdateNullTime(boolean isXValueNull) { return false; } + private void addInput(Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { + int selectPositionCount = mask.getSelectedPositionCount(); + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; + } + + boolean isXValueNull = xColumn.isNull(position); + if (!timeColumn.isNull(position)) { + if (checkAndUpdateFirstTime(isXValueNull, timeColumn.getLong(position)) && !isXValueNull) { + xValueSetter.set(xResult, xColumn, position); + } + if (canFinishAfterInit) { + return; + } + } else if (checkAndUpdateNullTime(isXValueNull) && !isXValueNull) { + xValueSetter.set(xResult, xColumn, position); + } + } + } + protected void addIntInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { int selectPositionCount = mask.getSelectedPositionCount(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/LastAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/LastAccumulator.java index 5918a640c303..90f2d2785665 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/LastAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/LastAccumulator.java @@ -20,6 +20,7 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -29,6 +30,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -40,11 +42,15 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.Utils.serializeTimeValueWithNull; -public class LastAccumulator implements TableAccumulator { +public class LastAccumulator implements TableAccumulator, TypeServices.LastValueUpdater { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(LastAccumulator.class); protected final TSDataType seriesDataType; + protected final Type type; + private final TypeServices.PrimitiveColumnValueSetter valueSetter; + private final TypeServices.StatisticsValueSetter statisticsValueSetter; + private final TypeServices.LastValueDeserializer valueDeserializer; protected TsPrimitiveType lastValue; protected long maxTime = Long.MIN_VALUE; protected boolean initResult = false; @@ -52,7 +58,11 @@ public class LastAccumulator implements TableAccumulator { public LastAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - this.lastValue = TsPrimitiveType.getByType(seriesDataType); + this.type = Type.fromTsDataType(seriesDataType); + this.lastValue = type.getTsPrimitiveType(); + this.valueSetter = TypeServices.PRIMITIVE_COLUMN_VALUE_SETTER_SERVICE.call(type); + this.statisticsValueSetter = TypeServices.STATISTICS_VALUE_SETTER_SERVICE.call(type); + this.valueDeserializer = TypeServices.LAST_VALUE_DESERIALIZER_SERVICE.call(type); } public boolean hasInitResult() { @@ -80,35 +90,7 @@ public TableAccumulator copy() { @Override public void addInput(Column[] arguments, AggregationMask mask) { // arguments[0] is value column, arguments[1] is time column - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(arguments[0], arguments[1], mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(arguments[0], arguments[1], mask); - return; - case FLOAT: - addFloatInput(arguments[0], arguments[1], mask); - return; - case DOUBLE: - addDoubleInput(arguments[0], arguments[1], mask); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(arguments[0], arguments[1], mask); - return; - case BOOLEAN: - addBooleanInput(arguments[0], arguments[1], mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_ARG_37F52124, seriesDataType)); - } + addInput(arguments[0], arguments[1], mask); } @Override @@ -129,69 +111,7 @@ public void addIntermediate(Column argument) { long time = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); boolean isOrderTimeNull = BytesUtils.bytesToBool(bytes, Long.BYTES); int offset = Long.BYTES + 1; - - switch (seriesDataType) { - case INT32: - case DATE: - int intVal = BytesUtils.bytesToInt(bytes, offset); - if (!isOrderTimeNull) { - updateIntLastValue(intVal, time); - } else { - updateIntNullTimeValue(intVal); - } - break; - case INT64: - case TIMESTAMP: - long longVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - if (!isOrderTimeNull) { - updateLongLastValue(longVal, time); - } else { - updateLongNullTimeValue(longVal); - } - break; - case FLOAT: - float floatVal = BytesUtils.bytesToFloat(bytes, offset); - if (!isOrderTimeNull) { - updateFloatLastValue(floatVal, time); - } else { - updateFloatNullTimeValue(floatVal); - } - break; - case DOUBLE: - double doubleVal = BytesUtils.bytesToDouble(bytes, offset); - if (!isOrderTimeNull) { - updateDoubleLastValue(doubleVal, time); - } else { - updateDoubleNullTimeValue(doubleVal); - } - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - if (!isOrderTimeNull) { - updateBinaryLastValue(binaryVal, time); - } else { - updateBinaryNullTimeValue(binaryVal); - } - break; - case BOOLEAN: - boolean boolVal = BytesUtils.bytesToBool(bytes, offset); - if (!isOrderTimeNull) { - updateBooleanLastValue(boolVal, time); - } else { - updateBooleanNullTimeValue(boolVal); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_2FCF5ADA, - seriesDataType)); - } + valueDeserializer.deserialize(bytes, offset, time, isOrderTimeNull, this); } } @@ -222,36 +142,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(lastValue.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(lastValue.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(lastValue.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(lastValue.getDouble()); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - columnBuilder.writeBinary(lastValue.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(lastValue.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_B3F2EE42, - seriesDataType)); - } + type.write(columnBuilder, lastValue); } @Override @@ -259,60 +150,103 @@ public boolean hasFinalResult() { return false; } + private void addInput(Column valueColumn, Column timeColumn, AggregationMask mask) { + int selectPositionCount = mask.getSelectedPositionCount(); + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; + } + + if (!timeColumn.isNull(position)) { + if (checkAndUpdateLastTime(timeColumn.getLong(position))) { + valueSetter.set(lastValue, valueColumn, position); + } + } else if (checkAndUpdateNullTime()) { + valueSetter.set(lastValue, valueColumn, position); + } + } + } + + @Override + public void updateInt(int value, long time, boolean isOrderTimeNull) { + if (isOrderTimeNull) { + updateIntNullTimeValue(value); + } else { + updateIntLastValue(value, time); + } + } + + @Override + public void updateLong(long value, long time, boolean isOrderTimeNull) { + if (isOrderTimeNull) { + updateLongNullTimeValue(value); + } else { + updateLongLastValue(value, time); + } + } + + @Override + public void updateFloat(float value, long time, boolean isOrderTimeNull) { + if (isOrderTimeNull) { + updateFloatNullTimeValue(value); + } else { + updateFloatLastValue(value, time); + } + } + + @Override + public void updateDouble(double value, long time, boolean isOrderTimeNull) { + if (isOrderTimeNull) { + updateDoubleNullTimeValue(value); + } else { + updateDoubleLastValue(value, time); + } + } + + @Override + public void updateBinary(Binary value, long time, boolean isOrderTimeNull) { + if (isOrderTimeNull) { + updateBinaryNullTimeValue(value); + } else { + updateBinaryLastValue(value, time); + } + } + + @Override + public void updateBoolean(boolean value, long time, boolean isOrderTimeNull) { + if (isOrderTimeNull) { + updateBooleanNullTimeValue(value); + } else { + updateBooleanLastValue(value, time); + } + } + @Override public void addStatistics(Statistics[] statistics) { if (statistics == null || statistics[0] == null) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntLastValue( - ((Number) statistics[0].getLastValue()).intValue(), statistics[0].getEndTime()); - break; - case INT64: - case TIMESTAMP: - updateLongLastValue( - ((Number) statistics[0].getLastValue()).longValue(), statistics[0].getEndTime()); - break; - case FLOAT: - updateFloatLastValue( - ((Number) statistics[0].getLastValue()).floatValue(), statistics[0].getEndTime()); - break; - case DOUBLE: - updateDoubleLastValue( - ((Number) statistics[0].getLastValue()).doubleValue(), statistics[0].getEndTime()); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - if (statistics[0] instanceof DateStatistics) { - updateBinaryLastValue( - new Binary( - TSDataType.getDateStringValue((Integer) statistics[0].getLastValue()), - StandardCharsets.UTF_8), - statistics[0].getEndTime()); - } else { - if (statistics[0].getLastValue() instanceof Binary) { - updateBinaryLastValue( - (Binary) statistics[0].getLastValue(), statistics[0].getEndTime()); - } else { - updateBinaryLastValue( - new Binary(String.valueOf(statistics[0].getLastValue()), StandardCharsets.UTF_8), - statistics[0].getEndTime()); - } - } - break; - case BOOLEAN: - updateBooleanLastValue((boolean) statistics[0].getLastValue(), statistics[0].getEndTime()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_2FCF5ADA, - seriesDataType)); + Object value = statistics[0].getLastValue(); + if (statistics[0] instanceof DateStatistics + && (seriesDataType == TSDataType.TEXT + || seriesDataType == TSDataType.STRING + || seriesDataType == TSDataType.BLOB + || seriesDataType == TSDataType.OBJECT)) { + value = new Binary(TSDataType.getDateStringValue((Integer) value), StandardCharsets.UTF_8); + } + if (checkAndUpdateLastTime(statistics[0].getEndTime())) { + statisticsValueSetter.set( + lastValue, + value, + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_LAST_AGGREGATION, seriesDataType))); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/LastByAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/LastByAccumulator.java index c5dfd41eacbb..b57161b072d1 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/LastByAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/LastByAccumulator.java @@ -20,6 +20,7 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -28,12 +29,15 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.nio.ByteBuffer; + import static com.google.common.base.Preconditions.checkArgument; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.Utils.serializeTimeValueWithNull; @@ -50,6 +54,9 @@ public class LastByAccumulator implements TableAccumulator { private long yLastTime = Long.MIN_VALUE; + private final Type xType; + private final TypeServices.PrimitiveColumnValueSetter xValueSetter; + private final TypeServices.StatisticsValueSetter xStatisticsValueSetter; private final TsPrimitiveType xResult; private boolean xIsNull = true; @@ -62,7 +69,10 @@ public LastByAccumulator( this.yDataType = yDataType; this.xIsTimeColumn = xIsTimeColumn; this.yIsTimeColumn = yIsTimeColumn; - this.xResult = TsPrimitiveType.getByType(xDataType); + this.xType = Type.fromTsDataType(xDataType); + this.xValueSetter = TypeServices.PRIMITIVE_COLUMN_VALUE_SETTER_SERVICE.call(xType); + this.xStatisticsValueSetter = TypeServices.STATISTICS_VALUE_SETTER_SERVICE.call(xType); + this.xResult = xType.getTsPrimitiveType(); } public boolean hasInitResult() { @@ -99,36 +109,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { .EXCEPTION_LENGTH_OF_INPUT_COLUMN_LEFT_BRACKET_RIGHT_BRACKET_FOR_LAST_BY_SHOULD_BE_3_D759D45F); // arguments[0] is x column, arguments[1] is y column, arguments[2] is time column - switch (xDataType) { - case INT32: - case DATE: - addIntInput(arguments[0], arguments[1], arguments[2], mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(arguments[0], arguments[1], arguments[2], mask); - return; - case FLOAT: - addFloatInput(arguments[0], arguments[1], arguments[2], mask); - return; - case DOUBLE: - addDoubleInput(arguments[0], arguments[1], arguments[2], mask); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(arguments[0], arguments[1], arguments[2], mask); - return; - case BOOLEAN: - addBooleanInput(arguments[0], arguments[1], arguments[2], mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - yDataType)); - } + addInput(arguments[0], arguments[1], arguments[2], mask); } @Override @@ -153,81 +134,16 @@ public void addIntermediate(Column argument) { boolean isXValueNull = BytesUtils.bytesToBool(bytes, offset); offset += 1; - switch (xDataType) { - case INT32: - case DATE: - // if the x value is null, could not serialize the x value - int xIntVal = isXValueNull ? 0 : BytesUtils.bytesToInt(bytes, offset); - if (!isOrderTimeNull) { - updateIntLastValue(isXValueNull, xIntVal, curTime); - } else { - updateIntNullTimeValue(isXValueNull, xIntVal); - } - break; - - case INT64: - case TIMESTAMP: - long longVal = - isXValueNull ? 0 : BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - if (!isOrderTimeNull) { - updateLongLastValue(isXValueNull, longVal, curTime); - } else { - updateLongNullTimeValue(isXValueNull, longVal); - } - break; - - case FLOAT: - float floatVal = isXValueNull ? 0 : BytesUtils.bytesToFloat(bytes, offset); - if (!isOrderTimeNull) { - updateFloatLastValue(isXValueNull, floatVal, curTime); - } else { - updateFloatNullTimeValue(isXValueNull, floatVal); - } - break; - - case DOUBLE: - double doubleVal = isXValueNull ? 0 : BytesUtils.bytesToDouble(bytes, offset); - if (!isOrderTimeNull) { - updateDoubleLastValue(isXValueNull, doubleVal, curTime); - } else { - updateDoubleNullTimeValue(isXValueNull, doubleVal); - } - break; - - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary binaryVal = null; - if (!isXValueNull) { - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - } - if (!isOrderTimeNull) { - updateBinaryLastValue(isXValueNull, binaryVal, curTime); - } else { - updateBinaryNullTimeValue(isXValueNull, binaryVal); - } - break; - - case BOOLEAN: - boolean boolVal = false; - if (!isXValueNull) { - boolVal = BytesUtils.bytesToBool(bytes, offset); - } - if (!isOrderTimeNull) { - updateBooleanLastValue(isXValueNull, boolVal, curTime); - } else { - updateBooleanNullTimeValue(isXValueNull, boolVal); - } - break; - - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - yDataType)); + TsPrimitiveType value = + isXValueNull + ? null + : xType.deserialize(ByteBuffer.wrap(bytes, offset, bytes.length - offset)); + if (!isOrderTimeNull) { + if (checkAndUpdateLastTime(isXValueNull, curTime) && !isXValueNull) { + xResult.copy(value); + } + } else if (checkAndUpdateNullTime(isXValueNull) && !isXValueNull) { + xResult.copy(value); } } } @@ -259,36 +175,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { return; } - switch (xDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(xResult.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(xResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(xResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(xResult.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - columnBuilder.writeBinary(xResult.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(xResult.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - xDataType)); - } + xType.write(columnBuilder, xResult); } @Override @@ -321,36 +208,13 @@ public void addStatistics(Statistics[] statistics) { yLastTime = yStatistics.getEndTime(); xIsNull = false; - switch (xDataType) { - case INT32: - case DATE: - xResult.setInt(((Number) xStatistics.getLastValue()).intValue()); - break; - case INT64: - case TIMESTAMP: - xResult.setLong(((Number) xStatistics.getLastValue()).longValue()); - break; - case FLOAT: - xResult.setFloat(((Number) statistics[0].getLastValue()).floatValue()); - break; - case DOUBLE: - xResult.setDouble(((Number) statistics[0].getLastValue()).doubleValue()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - xResult.setBinary((Binary) statistics[0].getLastValue()); - break; - case BOOLEAN: - xResult.setBoolean((boolean) statistics[0].getLastValue()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - yDataType)); - } + xStatisticsValueSetter.set( + xResult, + xStatistics.getLastValue(), + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_LAST_BY_AGGREGATION, yDataType))); } } } else { @@ -373,6 +237,28 @@ public void reset() { this.xResult.reset(); } + private void addInput(Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { + int selectPositionCount = mask.getSelectedPositionCount(); + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; + } + + boolean isXValueNull = xColumn.isNull(position); + if (!timeColumn.isNull(position)) { + if (checkAndUpdateLastTime(isXValueNull, timeColumn.getLong(position)) && !isXValueNull) { + xValueSetter.set(xResult, xColumn, position); + } + } else if (checkAndUpdateNullTime(isXValueNull) && !isXValueNull) { + xValueSetter.set(xResult, xColumn, position); + } + } + } + private boolean checkAndUpdateLastTime(boolean isXValueNull, long curTime) { if (!initResult || curTime > yLastTime) { initResult = true; diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/MaxAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/MaxAccumulator.java index 37047146d011..42020fda21df 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/MaxAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/MaxAccumulator.java @@ -20,18 +20,19 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.statistics.BinaryStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; -import java.nio.charset.StandardCharsets; +import java.util.function.Supplier; import static com.google.common.base.Preconditions.checkArgument; @@ -39,12 +40,18 @@ public class MaxAccumulator implements TableAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(MaxAccumulator.class); private final TSDataType seriesDataType; + private final Type type; + private final TypeServices.ColumnValueUpdater valueUpdater; + private final TypeServices.StatisticsValueUpdater statisticsValueUpdater; private final TsPrimitiveType maxResult; private boolean initResult; public MaxAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - this.maxResult = TsPrimitiveType.getByType(seriesDataType); + this.type = Type.fromTsDataType(seriesDataType); + this.maxResult = type.getTsPrimitiveType(); + this.valueUpdater = TypeServices.MAX_COLUMN_VALUE_UPDATER_SERVICE.call(type); + this.statisticsValueUpdater = TypeServices.MAX_STATISTICS_VALUE_UPDATER_SERVICE.call(type); } @Override @@ -63,35 +70,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { arguments.length == 1, CalcMessages.EXCEPTION_ARGUMENT_OF_MAX_SHOULD_BE_ONE_COLUMN_FC251F55); - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(arguments[0], mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(arguments[0], mask); - return; - case FLOAT: - addFloatInput(arguments[0], mask); - return; - case DOUBLE: - addDoubleInput(arguments[0], mask); - return; - case TEXT: - case STRING: - case BLOB: - addBinaryInput(arguments[0], mask); - return; - case BOOLEAN: - addBooleanInput(arguments[0], mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); - } + addInput(arguments[0], mask); } @Override @@ -101,34 +80,8 @@ public void addIntermediate(Column argument) { continue; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntMaxValue(argument.getInt(i)); - break; - case INT64: - case TIMESTAMP: - updateLongMaxValue(argument.getLong(i)); - break; - case FLOAT: - updateFloatMaxValue(argument.getFloat(i)); - break; - case DOUBLE: - updateDoubleMaxValue(argument.getDouble(i)); - break; - case STRING: - case TEXT: - case BLOB: - updateBinaryMaxValue(argument.getBinary(i)); - break; - case BOOLEAN: - updateBooleanMaxValue(argument.getBoolean(i)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); + if (valueUpdater.update(maxResult, argument, i, initResult)) { + initResult = true; } } } @@ -140,35 +93,7 @@ public void evaluateIntermediate(ColumnBuilder columnBuilder) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(maxResult.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(maxResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(maxResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(maxResult.getDouble()); - break; - case STRING: - case TEXT: - case BLOB: - columnBuilder.writeBinary(maxResult.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(maxResult.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); - } + type.write(columnBuilder, maxResult); } @Override @@ -177,35 +102,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(maxResult.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(maxResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(maxResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(maxResult.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - columnBuilder.writeBinary(maxResult.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(maxResult.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); - } + type.write(columnBuilder, maxResult); } @Override @@ -218,43 +115,14 @@ public void addStatistics(Statistics[] statistics) { if (statistics == null || statistics[0] == null) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntMaxValue(((Number) statistics[0].getMaxValue()).intValue()); - break; - case INT64: - case TIMESTAMP: - updateLongMaxValue(((Number) statistics[0].getMaxValue()).longValue()); - break; - case FLOAT: - updateFloatMaxValue(((Number) statistics[0].getMaxValue()).floatValue()); - break; - case DOUBLE: - updateDoubleMaxValue(((Number) statistics[0].getMaxValue()).doubleValue()); - break; - case TEXT: - case BLOB: - case STRING: - if (statistics[0] instanceof BinaryStatistics) { - updateBinaryMaxValue((Binary) statistics[0].getMaxValue()); - } else { - if (statistics[0].getMaxValue() instanceof Binary) { - updateBinaryMaxValue((Binary) statistics[0].getMaxValue()); - } else { - updateBinaryMaxValue( - new Binary(String.valueOf(statistics[0].getMaxValue()), StandardCharsets.UTF_8)); - } - } - break; - case BOOLEAN: - updateBooleanMaxValue((boolean) statistics[0].getMaxValue()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); + Supplier exceptionSupplier = + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_MAX_AGGREGATION, seriesDataType)); + if (statisticsValueUpdater.update( + maxResult, statistics[0].getMaxValue(), initResult, exceptionSupplier)) { + initResult = true; } } @@ -264,6 +132,26 @@ public void reset() { this.maxResult.reset(); } + private void addInput(Column valueColumn, AggregationMask mask) { + int positionCount = mask.getSelectedPositionCount(); + if (mask.isSelectAll()) { + for (int i = 0; i < positionCount; i++) { + if (!valueColumn.isNull(i) && valueUpdater.update(maxResult, valueColumn, i, initResult)) { + initResult = true; + } + } + } else { + int[] selectedPositions = mask.getSelectedPositions(); + for (int i = 0; i < positionCount; i++) { + int position = selectedPositions[i]; + if (!valueColumn.isNull(position) + && valueUpdater.update(maxResult, valueColumn, position, initResult)) { + initResult = true; + } + } + } + } + private void addIntInput(Column valueColumn, AggregationMask mask) { int positionCount = mask.getSelectedPositionCount(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/MinAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/MinAccumulator.java index 08f99b53b035..9a657d31347d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/MinAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/MinAccumulator.java @@ -20,18 +20,19 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.statistics.BinaryStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; -import java.nio.charset.StandardCharsets; +import java.util.function.Supplier; import static com.google.common.base.Preconditions.checkArgument; @@ -39,12 +40,18 @@ public class MinAccumulator implements TableAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(MinAccumulator.class); private final TSDataType seriesDataType; + private final Type type; + private final TypeServices.ColumnValueUpdater valueUpdater; + private final TypeServices.StatisticsValueUpdater statisticsValueUpdater; private final TsPrimitiveType minResult; private boolean initResult; public MinAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - this.minResult = TsPrimitiveType.getByType(seriesDataType); + this.type = Type.fromTsDataType(seriesDataType); + this.minResult = type.getTsPrimitiveType(); + this.valueUpdater = TypeServices.MIN_COLUMN_VALUE_UPDATER_SERVICE.call(type); + this.statisticsValueUpdater = TypeServices.MIN_STATISTICS_VALUE_UPDATER_SERVICE.call(type); } @Override @@ -63,35 +70,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { arguments.length == 1, CalcMessages.EXCEPTION_ARGUMENT_OF_MIN_SHOULD_BE_ONE_COLUMN_F8AA1E88); - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(arguments[0], mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(arguments[0], mask); - return; - case FLOAT: - addFloatInput(arguments[0], mask); - return; - case DOUBLE: - addDoubleInput(arguments[0], mask); - return; - case TEXT: - case STRING: - case BLOB: - addBinaryInput(arguments[0], mask); - return; - case BOOLEAN: - addBooleanInput(arguments[0], mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); - } + addInput(arguments[0], mask); } @Override @@ -101,34 +80,8 @@ public void addIntermediate(Column argument) { continue; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntMinValue(argument.getInt(i)); - break; - case INT64: - case TIMESTAMP: - updateLongMinValue(argument.getLong(i)); - break; - case FLOAT: - updateFloatMinValue(argument.getFloat(i)); - break; - case DOUBLE: - updateDoubleMinValue(argument.getDouble(i)); - break; - case STRING: - case TEXT: - case BLOB: - updateBinaryMinValue(argument.getBinary(i)); - break; - case BOOLEAN: - updateBooleanMinValue(argument.getBoolean(i)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); + if (valueUpdater.update(minResult, argument, i, initResult)) { + initResult = true; } } } @@ -140,35 +93,7 @@ public void evaluateIntermediate(ColumnBuilder columnBuilder) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(minResult.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(minResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(minResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(minResult.getDouble()); - break; - case STRING: - case TEXT: - case BLOB: - columnBuilder.writeBinary(minResult.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(minResult.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); - } + type.write(columnBuilder, minResult); } @Override @@ -177,35 +102,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(minResult.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(minResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(minResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(minResult.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - columnBuilder.writeBinary(minResult.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(minResult.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); - } + type.write(columnBuilder, minResult); } @Override @@ -218,43 +115,14 @@ public void addStatistics(Statistics[] statistics) { if (statistics == null || statistics[0] == null) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntMinValue(((Number) statistics[0].getMinValue()).intValue()); - break; - case INT64: - case TIMESTAMP: - updateLongMinValue(((Number) statistics[0].getMinValue()).longValue()); - break; - case FLOAT: - updateFloatMinValue(((Number) statistics[0].getMinValue()).floatValue()); - break; - case DOUBLE: - updateDoubleMinValue(((Number) statistics[0].getMinValue()).doubleValue()); - break; - case TEXT: - case BLOB: - case STRING: - if (statistics[0] instanceof BinaryStatistics) { - updateBinaryMinValue((Binary) statistics[0].getMinValue()); - } else { - if (statistics[0].getMinValue() instanceof Binary) { - updateBinaryMinValue((Binary) statistics[0].getMinValue()); - } else { - updateBinaryMinValue( - new Binary(String.valueOf(statistics[0].getMinValue()), StandardCharsets.UTF_8)); - } - } - break; - case BOOLEAN: - updateBooleanMinValue((boolean) statistics[0].getMinValue()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); + Supplier exceptionSupplier = + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_MIN_AGGREGATION, seriesDataType)); + if (statisticsValueUpdater.update( + minResult, statistics[0].getMinValue(), initResult, exceptionSupplier)) { + initResult = true; } } @@ -264,6 +132,26 @@ public void reset() { this.minResult.reset(); } + private void addInput(Column valueColumn, AggregationMask mask) { + int positionCount = mask.getSelectedPositionCount(); + if (mask.isSelectAll()) { + for (int i = 0; i < positionCount; i++) { + if (!valueColumn.isNull(i) && valueUpdater.update(minResult, valueColumn, i, initResult)) { + initResult = true; + } + } + } else { + int[] selectedPositions = mask.getSelectedPositions(); + for (int i = 0; i < positionCount; i++) { + int position = selectedPositions[i]; + if (!valueColumn.isNull(position) + && valueUpdater.update(minResult, valueColumn, position, initResult)) { + initResult = true; + } + } + } + } + private void addIntInput(Column valueColumn, AggregationMask mask) { int positionCount = mask.getSelectedPositionCount(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/PercentileAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/PercentileAccumulator.java index 531e2bd0bbfd..e625c2876aa1 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/PercentileAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/PercentileAccumulator.java @@ -22,12 +22,14 @@ import org.apache.iotdb.calc.execution.operator.source.relational.Percentile; import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -40,6 +42,8 @@ public class PercentileAccumulator implements TableAccumulator { RamUsageEstimator.shallowSizeOfInstance(PercentileAccumulator.class); private final TSDataType seriesDataType; + private final TypeServices.ColumnToDoubleConverter valueConverter; + private final TypeServices.NumericResultWriter resultWriter; private Percentile percentile = new Percentile(); // percentage is a query-level constant; it is read once from the first input/intermediate and // kept fixed afterwards, so it never gets reset to 0 by a later all-null batch. @@ -53,6 +57,17 @@ public PercentileAccumulator( TSDataType seriesDataType, MemoryReservationManager memoryReservationManager) { this.seriesDataType = seriesDataType; this.memoryReservationManager = memoryReservationManager; + Type type = Type.fromTsDataType(seriesDataType); + this.valueConverter = + TypeServices.PERCENTILE_NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(type) + .create( + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_PERCENTILE_AGGREGATION, + seriesDataType))); + this.resultWriter = TypeServices.NUMERIC_RESULT_WRITER_SERVICE.call(type); updateMemoryReservation(); } @@ -70,34 +85,13 @@ public TableAccumulator copy() { public void addInput(Column[] arguments, AggregationMask mask) { if (arguments.length != 2) { throw new SemanticException( - String.format( - CalcMessages.EXCEPTION_PERCENTILE_REQUIRES_2_ARGUMENTS_BUT_GOT_ARG_F3F1882F, - arguments.length)); + String.format(CalcMessages.PERCENTILE_REQUIRES_TWO_ARGUMENTS, arguments.length)); } if (!percentageInitialized) { percentage = arguments[1].getDouble(0); percentageInitialized = true; } - switch (seriesDataType) { - case INT32: - addIntInput(arguments[0], mask); - break; - case INT64: - case TIMESTAMP: - addLongInput(arguments[0], mask); - break; - case FLOAT: - addFloatInput(arguments[0], mask); - break; - case DOUBLE: - addDoubleInput(arguments[0], mask); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_PERCENTILE_AGGREGATION_ARG_E0C2513D, - seriesDataType)); - } + addInput(arguments[0], mask); updateMemoryReservation(); } @@ -137,26 +131,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (seriesDataType) { - case INT32: - columnBuilder.writeInt((int) result); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong((long) result); - break; - case FLOAT: - columnBuilder.writeFloat((float) result); - break; - case DOUBLE: - columnBuilder.writeDouble(result); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_PERCENTILE_AGGREGATION_ARG_8ACED24D, - seriesDataType)); - } + resultWriter.write(columnBuilder, result); } @Override @@ -167,7 +142,7 @@ public boolean hasFinalResult() { @Override public void addStatistics(Statistics[] statistics) { throw new UnsupportedOperationException( - CalcMessages.EXCEPTION_PERCENTILEACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS_66308C79); + CalcMessages.PERCENTILE_ACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS); } @Override @@ -188,76 +163,13 @@ private void updateMemoryReservation() { previousPercentileSize = currentSize; } - private void addIntInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - percentile.addValue(column.getInt(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - percentile.addValue(column.getInt(position)); - } - } - } - } - - private void addLongInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - percentile.addValue(column.getLong(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - percentile.addValue(column.getLong(position)); - } - } - } - } - - private void addFloatInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - percentile.addValue(column.getFloat(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - percentile.addValue(column.getFloat(position)); - } - } - } - } - - private void addDoubleInput(Column column, AggregationMask mask) { + private void addInput(Column column, AggregationMask mask) { int positionCount = mask.getSelectedPositionCount(); if (mask.isSelectAll()) { for (int i = 0; i < positionCount; i++) { if (!column.isNull(i)) { - percentile.addValue(column.getDouble(i)); + percentile.addValue(valueConverter.convert(column, i)); } } } else { @@ -266,7 +178,7 @@ private void addDoubleInput(Column column, AggregationMask mask) { for (int i = 0; i < positionCount; i++) { position = selectedPositions[i]; if (!column.isNull(position)) { - percentile.addValue(column.getDouble(position)); + percentile.addValue(valueConverter.convert(column, position)); } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/SumAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/SumAccumulator.java index 98fee0156072..b375bf466fd5 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/SumAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/SumAccumulator.java @@ -19,12 +19,14 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.IntegerStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -34,11 +36,22 @@ public class SumAccumulator implements TableAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(SumAccumulator.class); private final TSDataType argumentDataType; + private final TypeServices.ColumnToDoubleConverter valueConverter; private double sumValue = 0; private boolean initResult = false; public SumAccumulator(TSDataType argumentDataType) { this.argumentDataType = argumentDataType; + Type type = Type.fromTsDataType(argumentDataType); + this.valueConverter = + TypeServices.AGGREGATION_NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(type) + .create( + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_SUM_AGGREGATION, + argumentDataType))); } @Override @@ -53,68 +66,14 @@ public TableAccumulator copy() { @Override public void addInput(Column[] arguments, AggregationMask mask) { - checkArgument( - arguments.length == 1, - CalcMessages.EXCEPTION_ARGUMENT_OF_SUM_SHOULD_BE_ONE_COLUMN_D6E636D1); - switch (argumentDataType) { - case INT32: - addIntInput(arguments[0], mask); - return; - case INT64: - addLongInput(arguments[0], mask); - return; - case FLOAT: - addFloatInput(arguments[0], mask); - return; - case DOUBLE: - addDoubleInput(arguments[0], mask); - return; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_SUM_AGGREGATION_ARG_92F5A18D, - argumentDataType)); - } + checkArgument(arguments.length == 1, "argument of SUM should be one column"); + addInput(arguments[0], mask); } @Override public void removeInput(Column[] arguments) { - checkArgument( - arguments.length == 1, - CalcMessages.EXCEPTION_ARGUMENT_OF_SUM_SHOULD_BE_ONE_COLUMN_D6E636D1); - switch (argumentDataType) { - case INT32: - removeIntInput(arguments[0]); - return; - case INT64: - removeLongInput(arguments[0]); - return; - case FLOAT: - removeFloatInput(arguments[0]); - return; - case DOUBLE: - removeDoubleInput(arguments[0]); - return; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_SUM_AGGREGATION_ARG_92F5A18D, - argumentDataType)); - } + checkArgument(arguments.length == 1, "argument of SUM should be one column"); + removeInput(arguments[0]); } @Override @@ -177,79 +136,7 @@ public boolean removable() { return true; } - private void addIntInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - int count = column.getPositionCount(); - for (int i = 0; i < count; i++) { - if (!column.isNull(i)) { - initResult = true; - sumValue += column.getInt(i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - initResult = true; - sumValue += column.getInt(position); - } - } - } - } - - private void addLongInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - int count = column.getPositionCount(); - for (int i = 0; i < count; i++) { - if (!column.isNull(i)) { - initResult = true; - sumValue += column.getLong(i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - initResult = true; - sumValue += column.getLong(position); - } - } - } - } - - private void addFloatInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - int count = column.getPositionCount(); - for (int i = 0; i < count; i++) { - if (!column.isNull(i)) { - initResult = true; - sumValue += column.getFloat(i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - initResult = true; - sumValue += column.getFloat(position); - } - } - } - } - - private void addDoubleInput(Column column, AggregationMask mask) { + private void addInput(Column column, AggregationMask mask) { int positionCount = mask.getSelectedPositionCount(); if (mask.isSelectAll()) { @@ -257,7 +144,7 @@ private void addDoubleInput(Column column, AggregationMask mask) { for (int i = 0; i < count; i++) { if (!column.isNull(i)) { initResult = true; - sumValue += column.getDouble(i); + sumValue += valueConverter.convert(column, i); } } } else { @@ -267,44 +154,17 @@ private void addDoubleInput(Column column, AggregationMask mask) { position = selectedPositions[i]; if (!column.isNull(position)) { initResult = true; - sumValue += column.getDouble(position); + sumValue += valueConverter.convert(column, position); } } } } - private void removeIntInput(Column column) { - int count = column.getPositionCount(); - for (int i = 0; i < count; i++) { - if (!column.isNull(i)) { - sumValue -= column.getInt(i); - } - } - } - - private void removeLongInput(Column column) { - int count = column.getPositionCount(); - for (int i = 0; i < count; i++) { - if (!column.isNull(i)) { - sumValue -= column.getLong(i); - } - } - } - - private void removeFloatInput(Column column) { - int count = column.getPositionCount(); - for (int i = 0; i < count; i++) { - if (!column.isNull(i)) { - sumValue -= column.getFloat(i); - } - } - } - - private void removeDoubleInput(Column column) { + private void removeInput(Column column) { int count = column.getPositionCount(); for (int i = 0; i < count; i++) { if (!column.isNull(i)) { - sumValue -= column.getDouble(i); + sumValue -= valueConverter.convert(column, i); } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableCentralMomentAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableCentralMomentAccumulator.java index 82e1c04225c5..85fdcb864260 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableCentralMomentAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableCentralMomentAccumulator.java @@ -16,6 +16,7 @@ import org.apache.iotdb.calc.execution.aggregation.CentralMomentAccumulator; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -24,6 +25,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -40,6 +42,7 @@ public class TableCentralMomentAccumulator implements TableAccumulator { private static final double EPSILON = 1e-12; private final TSDataType seriesDataType; + private final TypeServices.ColumnToDoubleConverter doubleValueConverter; private final CentralMomentAccumulator.MomentType momentType; private long count; @@ -51,6 +54,15 @@ public class TableCentralMomentAccumulator implements TableAccumulator { public TableCentralMomentAccumulator( TSDataType seriesDataType, CentralMomentAccumulator.MomentType momentType) { this.seriesDataType = seriesDataType; + this.doubleValueConverter = + TypeServices.NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(Type.fromTsDataType(seriesDataType)) + .create( + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_CENTRAL_MOMENT_AGGREGATION, + seriesDataType))); this.momentType = momentType; } @@ -60,7 +72,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { if (mask.isSelectAll()) { for (int i = 0; i < positionCount; i++) { if (!arguments[0].isNull(i)) { - update(getDoubleValue(arguments[0], i)); + update(doubleValueConverter.convert(arguments[0], i)); } } } else { @@ -68,32 +80,12 @@ public void addInput(Column[] arguments, AggregationMask mask) { for (int i = 0; i < positionCount; i++) { int position = selectedPositions[i]; if (!arguments[0].isNull(position)) { - update(getDoubleValue(arguments[0], position)); + update(doubleValueConverter.convert(arguments[0], position)); } } } } - private double getDoubleValue(Column column, int position) { - switch (seriesDataType) { - case INT32: - case DATE: - return column.getInt(position); - case INT64: - case TIMESTAMP: - return column.getLong(position); - case FLOAT: - return column.getFloat(position); - case DOUBLE: - return column.getDouble(position); - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_CENTRALMOMENT_AGGREGATION_ARG_74B9A3A8, - seriesDataType)); - } - } - private void update(double value) { long n1 = count; long n = n1 + 1; @@ -239,7 +231,7 @@ public void removeInput(Column[] arguments) { return; } - double value = getDoubleValue(arguments[0], 0); + double value = doubleValueConverter.convert(arguments[0], 0); if (count == 1) { reset(); return; diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableCorrelationAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableCorrelationAccumulator.java index 8a99e89d7183..de1fd9cac71b 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableCorrelationAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableCorrelationAccumulator.java @@ -25,7 +25,6 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; @@ -71,8 +70,8 @@ public void addInput(Column[] arguments, AggregationMask mask) { if (arguments[0].isNull(i) || arguments[1].isNull(i)) { continue; } - double x = getDoubleValue(arguments[0], i, xDataType); - double y = getDoubleValue(arguments[1], i, yDataType); + double x = getDoubleValue(arguments[0], i); + double y = getDoubleValue(arguments[1], i); update(x, y); } } else { @@ -82,31 +81,15 @@ public void addInput(Column[] arguments, AggregationMask mask) { if (arguments[0].isNull(position) || arguments[1].isNull(position)) { continue; } - double x = getDoubleValue(arguments[0], position, xDataType); - double y = getDoubleValue(arguments[1], position, yDataType); + double x = getDoubleValue(arguments[0], position); + double y = getDoubleValue(arguments[1], position); update(x, y); } } } - private double getDoubleValue(Column column, int position, TSDataType dataType) { - switch (dataType) { - case INT32: - case DATE: - return column.getInt(position); - case INT64: - case TIMESTAMP: - return column.getLong(position); - case FLOAT: - return column.getFloat(position); - case DOUBLE: - return column.getDouble(position); - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_CORRELATION_AGGREGATION_ARG_2FF29597, - dataType)); - } + private double getDoubleValue(Column column, int position) { + return column.getDouble(position); } private void update(double x, double y) { @@ -130,8 +113,8 @@ public void removeInput(Column[] arguments) { return; } - double otherX = getDoubleValue(arguments[0], 0, xDataType); - double otherY = getDoubleValue(arguments[1], 0, yDataType); + double otherX = getDoubleValue(arguments[0], 0); + double otherY = getDoubleValue(arguments[1], 0); if (count == 1) { reset(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableCovarianceAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableCovarianceAccumulator.java index 0c07073c4434..8c179888d3a3 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableCovarianceAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableCovarianceAccumulator.java @@ -26,7 +26,6 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; @@ -76,8 +75,8 @@ public void addInput(Column[] arguments, AggregationMask mask) { if (arguments[0].isNull(i) || arguments[1].isNull(i)) { continue; } - double x = getDoubleValue(arguments[0], i, xDataType); - double y = getDoubleValue(arguments[1], i, yDataType); + double x = getDoubleValue(arguments[0], i); + double y = getDoubleValue(arguments[1], i); update(x, y); } } else { @@ -87,31 +86,15 @@ public void addInput(Column[] arguments, AggregationMask mask) { if (arguments[0].isNull(position) || arguments[1].isNull(position)) { continue; } - double x = getDoubleValue(arguments[0], position, xDataType); - double y = getDoubleValue(arguments[1], position, yDataType); + double x = getDoubleValue(arguments[0], position); + double y = getDoubleValue(arguments[1], position); update(x, y); } } } - private double getDoubleValue(Column column, int position, TSDataType dataType) { - switch (dataType) { - case INT32: - case DATE: - return column.getInt(position); - case INT64: - case TIMESTAMP: - return column.getLong(position); - case FLOAT: - return column.getFloat(position); - case DOUBLE: - return column.getDouble(position); - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_COVARIANCE_AGGREGATION_ARG_643BEDD0, - dataType)); - } + private double getDoubleValue(Column column, int position) { + return column.getDouble(position); } private void update(double x, double y) { @@ -133,8 +116,8 @@ public void removeInput(Column[] arguments) { return; } - double otherX = getDoubleValue(arguments[0], 0, xDataType); - double otherY = getDoubleValue(arguments[1], 0, yDataType); + double otherX = getDoubleValue(arguments[0], 0); + double otherY = getDoubleValue(arguments[1], 0); if (count == 1) { reset(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java index f26159d3fa2d..75109069592e 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java @@ -20,28 +20,27 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.Statistics; -import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; - -import java.util.Collections; import static com.google.common.base.Preconditions.checkArgument; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.Utils.calcTypeSize; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.Utils.serializeValue; /** max(x,y) returns the value of x associated with the maximum value of y over all input values. */ -public abstract class TableMaxMinByBaseAccumulator implements TableAccumulator { +public abstract class TableMaxMinByBaseAccumulator + implements TableAccumulator, TypeServices.MaxMinByValueUpdater { protected final TSDataType xDataType; @@ -50,55 +49,29 @@ public abstract class TableMaxMinByBaseAccumulator implements TableAccumulator { private final TsPrimitiveType yExtremeValue; private final TsPrimitiveType xResult; + private final Type xType; + private final TypeServices.PrimitiveColumnValueSetter xValueSetter; private boolean xNull = true; private boolean initResult; - public static final String UNSUPPORTED_TYPE_MESSAGE = - "Unsupported data type in MAX_BY/MIN_BY: %s"; - protected TableMaxMinByBaseAccumulator(TSDataType xDataType, TSDataType yDataType) { this.xDataType = xDataType; this.yDataType = yDataType; - this.xResult = TsPrimitiveType.getByType(xDataType); - this.yExtremeValue = TsPrimitiveType.getByType(yDataType); + this.xType = Type.fromTsDataType(xDataType); + this.xResult = xType.getTsPrimitiveType(); + this.yExtremeValue = Type.fromTsDataType(yDataType).getTsPrimitiveType(); + this.xValueSetter = TypeServices.PRIMITIVE_COLUMN_VALUE_SETTER_SERVICE.call(xType); } // Column should be like: | x | y | @Override public void addInput(Column[] arguments, AggregationMask mask) { - checkArgument( - arguments.length == 2, - CalcMessages - .EXCEPTION_LENGTH_OF_INPUT_COLUMN_LEFT_BRACKET_RIGHT_BRACKET_FOR_MAX_BY_SLASH_MIN_BY_SHOULD_3767E905); - switch (yDataType) { - case INT32: - case DATE: - addIntInput(arguments, mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(arguments, mask); - return; - case FLOAT: - addFloatInput(arguments, mask); - return; - case DOUBLE: - addDoubleInput(arguments, mask); - return; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - addBinaryInput(arguments, mask); - return; - case BOOLEAN: - addBooleanInput(arguments, mask); - return; - default: - throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, yDataType)); - } + checkArgument(arguments.length == 2, "Length of input Column[] for MAX_BY/MIN_BY should be 2"); + TypeServices.MAX_MIN_BY_INPUT_SERVICE + .call(Type.fromTsDataType(yDataType)) + .add(arguments, mask, this); } @Override @@ -161,28 +134,8 @@ public boolean hasFinalResult() { return false; } - private void addIntInput(Column[] column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column[1].isNull(i)) { - updateIntResult(column[1].getInt(i), column[0], i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column[1].isNull(position)) { - updateIntResult(column[1].getInt(position), column[0], position); - } - } - } - } - - private void updateIntResult(int yValue, Column xColumn, int xIndex) { + @Override + public void updateInt(int yValue, Column xColumn, int xIndex) { if (!initResult || check(yValue, yExtremeValue.getInt())) { initResult = true; yExtremeValue.setInt(yValue); @@ -190,28 +143,8 @@ private void updateIntResult(int yValue, Column xColumn, int xIndex) { } } - private void addLongInput(Column[] column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column[1].isNull(i)) { - updateLongResult(column[1].getLong(i), column[0], i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column[1].isNull(position)) { - updateLongResult(column[1].getLong(position), column[0], position); - } - } - } - } - - private void updateLongResult(long yValue, Column xColumn, int xIndex) { + @Override + public void updateLong(long yValue, Column xColumn, int xIndex) { if (!initResult || check(yValue, yExtremeValue.getLong())) { initResult = true; yExtremeValue.setLong(yValue); @@ -219,28 +152,8 @@ private void updateLongResult(long yValue, Column xColumn, int xIndex) { } } - private void addFloatInput(Column[] column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column[1].isNull(i)) { - updateFloatResult(column[1].getFloat(i), column[0], i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column[1].isNull(position)) { - updateFloatResult(column[1].getFloat(position), column[0], position); - } - } - } - } - - private void updateFloatResult(float yValue, Column xColumn, int xIndex) { + @Override + public void updateFloat(float yValue, Column xColumn, int xIndex) { if (!initResult || check(yValue, yExtremeValue.getFloat())) { initResult = true; yExtremeValue.setFloat(yValue); @@ -248,28 +161,8 @@ private void updateFloatResult(float yValue, Column xColumn, int xIndex) { } } - private void addDoubleInput(Column[] column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column[1].isNull(i)) { - updateDoubleResult(column[1].getDouble(i), column[0], i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column[1].isNull(position)) { - updateDoubleResult(column[1].getDouble(position), column[0], position); - } - } - } - } - - private void updateDoubleResult(double yValue, Column xColumn, int xIndex) { + @Override + public void updateDouble(double yValue, Column xColumn, int xIndex) { if (!initResult || check(yValue, yExtremeValue.getDouble())) { initResult = true; yExtremeValue.setDouble(yValue); @@ -277,28 +170,8 @@ private void updateDoubleResult(double yValue, Column xColumn, int xIndex) { } } - private void addBinaryInput(Column[] column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column[1].isNull(i)) { - updateBinaryResult(column[1].getBinary(i), column[0], i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column[1].isNull(position)) { - updateBinaryResult(column[1].getBinary(position), column[0], position); - } - } - } - } - - private void updateBinaryResult(Binary yValue, Column xColumn, int xIndex) { + @Override + public void updateBinary(Binary yValue, Column xColumn, int xIndex) { if (!initResult || check(yValue, yExtremeValue.getBinary())) { initResult = true; yExtremeValue.setBinary(yValue); @@ -306,28 +179,8 @@ private void updateBinaryResult(Binary yValue, Column xColumn, int xIndex) { } } - private void addBooleanInput(Column[] column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column[1].isNull(i)) { - updateBooleanResult(column[1].getBoolean(i), column[0], i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column[1].isNull(position)) { - updateBooleanResult(column[1].getBoolean(position), column[0], position); - } - } - } - } - - private void updateBooleanResult(boolean yValue, Column xColumn, int xIndex) { + @Override + public void updateBoolean(boolean yValue, Column xColumn, int xIndex) { if (!initResult || check(yValue, yExtremeValue.getBoolean())) { initResult = true; yExtremeValue.setBoolean(yValue); @@ -340,33 +193,7 @@ private void writeX(ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (xDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(xResult.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(xResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(xResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(xResult.getDouble()); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - columnBuilder.writeBinary(xResult.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(xResult.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, xDataType)); - } + xType.write(columnBuilder, xResult); } private void updateX(Column xColumn, int xIndex) { @@ -374,34 +201,7 @@ private void updateX(Column xColumn, int xIndex) { xNull = true; } else { xNull = false; - switch (xDataType) { - case INT32: - case DATE: - xResult.setInt(xColumn.getInt(xIndex)); - break; - case INT64: - case TIMESTAMP: - xResult.setLong(xColumn.getLong(xIndex)); - break; - case FLOAT: - xResult.setFloat(xColumn.getFloat(xIndex)); - break; - case DOUBLE: - xResult.setDouble(xColumn.getDouble(xIndex)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - xResult.setBinary(xColumn.getBinary(xIndex)); - break; - case BOOLEAN: - xResult.setBoolean(xColumn.getBoolean(xIndex)); - break; - default: - throw new UnSupportedDataTypeException( - String.format(UNSUPPORTED_TYPE_MESSAGE, xDataType)); - } + xValueSetter.set(xResult, xColumn, xIndex); } } @@ -427,97 +227,11 @@ private byte[] serialize() { } private void updateFromBytesIntermediateInput(byte[] bytes) { - int offset = 0; - // Use Column to store x value - TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(xDataType)); - ColumnBuilder columnBuilder = builder.getValueColumnBuilders()[0]; - switch (yDataType) { - case INT32: - case DATE: - int intMaxVal = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateIntResult(intMaxVal, columnBuilder.build(), 0); - break; - case INT64: - case TIMESTAMP: - long longMaxVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - offset += Long.BYTES; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateLongResult(longMaxVal, columnBuilder.build(), 0); - break; - case FLOAT: - float floatMaxVal = BytesUtils.bytesToFloat(bytes, offset); - offset += Float.BYTES; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateFloatResult(floatMaxVal, columnBuilder.build(), 0); - break; - case DOUBLE: - double doubleMaxVal = BytesUtils.bytesToDouble(bytes, offset); - offset += Long.BYTES; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateDoubleResult(doubleMaxVal, columnBuilder.build(), 0); - break; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - Binary binaryMaxVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - offset += length; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateBinaryResult(binaryMaxVal, columnBuilder.build(), 0); - break; - case BOOLEAN: - boolean booleanMaxVal = BytesUtils.bytesToBool(bytes, offset); - offset += 1; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateBooleanResult(booleanMaxVal, columnBuilder.build(), 0); - break; - default: - throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, yDataType)); - } - } - - private void readXFromBytesIntermediateInput( - byte[] bytes, int offset, ColumnBuilder columnBuilder) { - boolean isXNull = BytesUtils.bytesToBool(bytes, offset); - offset += 1; - if (isXNull) { - columnBuilder.appendNull(); - } else { - switch (xDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(BytesUtils.bytesToInt(bytes, offset)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(BytesUtils.bytesToLongFromOffset(bytes, 8, offset)); - break; - case FLOAT: - columnBuilder.writeFloat(BytesUtils.bytesToFloat(bytes, offset)); - break; - case DOUBLE: - columnBuilder.writeDouble(BytesUtils.bytesToDouble(bytes, offset)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - columnBuilder.writeBinary(new Binary(BytesUtils.subBytes(bytes, offset, length))); - break; - case BOOLEAN: - columnBuilder.writeBoolean(BytesUtils.bytesToBool(bytes, offset)); - break; - default: - throw new UnSupportedDataTypeException( - String.format(UNSUPPORTED_TYPE_MESSAGE, xDataType)); - } - } + // Use a one-row column so the shared updater applies the same null and x-value semantics as + // regular input processing. + TypeServices.MAX_MIN_BY_INTERMEDIATE_INPUT_SERVICE + .call(Type.fromTsDataType(yDataType)) + .update(bytes, xType, xType.createColumnBuilder(null, 1), this); } /** diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableModeAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableModeAccumulator.java index af9635a4f42b..6fb7496445d4 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableModeAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableModeAccumulator.java @@ -20,6 +20,7 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.tsfile.block.column.Column; @@ -29,6 +30,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -37,10 +39,6 @@ import java.util.Map; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.Utils.UNSUPPORTED_TYPE_MESSAGE; -import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.Utils.serializeBinaryValue; -import static org.apache.tsfile.utils.BytesUtils.bytesToBool; -import static org.apache.tsfile.utils.BytesUtils.bytesToLongFromOffset; public class TableModeAccumulator implements TableAccumulator { @@ -49,44 +47,17 @@ public class TableModeAccumulator implements TableAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(TableModeAccumulator.class); private final TSDataType seriesDataType; - - private Map booleanCountMap; - private Map intCountMap; - private Map floatCountMap; - private Map longCountMap; - private Map doubleCountMap; - private Map binaryCountMap; + private final TypeServices.ModeValueService valueService; + private final Map countMap = new HashMap<>(); private long nullCount; public TableModeAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - switch (seriesDataType) { - case BOOLEAN: - booleanCountMap = new HashMap<>(); - break; - case INT32: - case DATE: - intCountMap = new HashMap<>(); - break; - case FLOAT: - floatCountMap = new HashMap<>(); - break; - case INT64: - case TIMESTAMP: - longCountMap = new HashMap<>(); - break; - case DOUBLE: - doubleCountMap = new HashMap<>(); - break; - case TEXT: - case STRING: - case BLOB: - binaryCountMap = new HashMap<>(); - break; - default: - throw new UnsupportedOperationException( - String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); + this.valueService = TypeServices.MODE_VALUE_SERVICE.call(Type.fromTsDataType(seriesDataType)); + if (seriesDataType == TSDataType.OBJECT) { + throw new UnsupportedOperationException( + String.format(CalcMessages.UNSUPPORTED_DATA_TYPE_IN_MODE_AGGREGATION, seriesDataType)); } } @@ -102,63 +73,30 @@ public TableAccumulator copy() { @Override public void addInput(Column[] arguments, AggregationMask mask) { - switch (seriesDataType) { - case BOOLEAN: - addBooleanInput(arguments[0], mask); - break; - case INT32: - case DATE: - addIntInput(arguments[0], mask); - break; - case FLOAT: - addFloatInput(arguments[0], mask); - break; - case INT64: - case TIMESTAMP: - addLongInput(arguments[0], mask); - break; - case DOUBLE: - addDoubleInput(arguments[0], mask); - break; - case TEXT: - case STRING: - case BLOB: - addBinaryInput(arguments[0], mask); - break; - default: - throw new UnsupportedOperationException( - String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); + Column column = arguments[0]; + int positionCount = mask.getSelectedPositionCount(); + if (mask.isSelectAll()) { + for (int i = 0; i < positionCount; i++) { + addValue(column, i); + } + } else { + int[] selectedPositions = mask.getSelectedPositions(); + for (int i = 0; i < positionCount; i++) { + addValue(column, selectedPositions[i]); + } } } @Override public void removeInput(Column[] arguments) { - switch (seriesDataType) { - case BOOLEAN: - removeBooleanInput(arguments[0]); - break; - case INT32: - case DATE: - removeIntInput(arguments[0]); - break; - case FLOAT: - removeFloatInput(arguments[0]); - break; - case INT64: - case TIMESTAMP: - removeLongInput(arguments[0]); - break; - case DOUBLE: - removeDoubleInput(arguments[0]); - break; - case TEXT: - case STRING: - case BLOB: - removeBinaryInput(arguments[0]); - break; - default: - throw new UnsupportedOperationException( - String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); + Column column = arguments[0]; + for (int i = 0; i < column.getPositionCount(); i++) { + if (column.isNull(i)) { + nullCount--; + } else { + Object key = valueService.getValue(column, i); + countMap.computeIfPresent(key, (k, count) -> count - 1); + } } } @@ -192,93 +130,16 @@ public void evaluateIntermediate(ColumnBuilder columnBuilder) { @Override public void evaluateFinal(ColumnBuilder columnBuilder) { - switch (seriesDataType) { - case BOOLEAN: - if (booleanCountMap.isEmpty()) { - columnBuilder.appendNull(); - } else { - // must be present - Map.Entry maxEntry = - booleanCountMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); - if (maxEntry.getValue() < nullCount) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeBoolean(maxEntry.getKey()); - } - } - break; - case INT32: - case DATE: - if (intCountMap.isEmpty()) { - columnBuilder.appendNull(); - } else { - Map.Entry maxEntry = - intCountMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); - if (maxEntry.getValue() < nullCount) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeInt(maxEntry.getKey()); - } - } - break; - case FLOAT: - if (floatCountMap.isEmpty()) { - columnBuilder.appendNull(); - } else { - Map.Entry maxEntry = - floatCountMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); - if (maxEntry.getValue() < nullCount) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeFloat(maxEntry.getKey()); - } - } - break; - case INT64: - case TIMESTAMP: - if (longCountMap.isEmpty()) { - columnBuilder.appendNull(); - } else { - Map.Entry maxEntry = - longCountMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); - if (maxEntry.getValue() < nullCount) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeLong(maxEntry.getKey()); - } - } - break; - case DOUBLE: - if (doubleCountMap.isEmpty()) { - columnBuilder.appendNull(); - } else { - Map.Entry maxEntry = - doubleCountMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); - if (maxEntry.getValue() < nullCount) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeDouble(maxEntry.getKey()); - } - } - break; - case TEXT: - case STRING: - case BLOB: - if (binaryCountMap.isEmpty()) { - columnBuilder.appendNull(); - } else { - Map.Entry maxEntry = - binaryCountMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); - if (maxEntry.getValue() < nullCount) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeBinary(maxEntry.getKey()); - } - } - break; - default: - throw new UnsupportedOperationException( - String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); + if (countMap.isEmpty()) { + columnBuilder.appendNull(); + return; + } + Map.Entry maxEntry = + countMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); + if (maxEntry.getValue() < nullCount) { + columnBuilder.appendNull(); + } else { + valueService.write(columnBuilder, maxEntry.getKey()); } } @@ -294,24 +155,7 @@ public void addStatistics(Statistics[] statistics) { @Override public void reset() { - if (booleanCountMap != null) { - booleanCountMap.clear(); - } - if (intCountMap != null) { - intCountMap.clear(); - } - if (floatCountMap != null) { - floatCountMap.clear(); - } - if (longCountMap != null) { - longCountMap.clear(); - } - if (doubleCountMap != null) { - doubleCountMap.clear(); - } - if (binaryCountMap != null) { - binaryCountMap.clear(); - } + countMap.clear(); nullCount = 0; } @@ -322,424 +166,53 @@ public boolean removable() { // haveNull | nullCount (optional) | countMap private byte[] serializeCountMap() { - byte[] bytes; int offset = 1 + (nullCount == 0 ? 0 : Long.BYTES); - ; - - switch (seriesDataType) { - case BOOLEAN: - bytes = new byte[offset + 4 + (1 + 8) * booleanCountMap.size()]; - BytesUtils.boolToBytes(nullCount != 0, bytes, 0); - if (nullCount != 0) { - BytesUtils.longToBytes(nullCount, bytes, 1); - } - BytesUtils.intToBytes(booleanCountMap.size(), bytes, offset); - offset += 4; - for (Map.Entry entry : booleanCountMap.entrySet()) { - BytesUtils.boolToBytes(entry.getKey(), bytes, offset); - offset += 1; - BytesUtils.longToBytes(entry.getValue(), bytes, offset); - offset += 8; - } - break; - case INT32: - case DATE: - bytes = new byte[offset + 4 + (4 + 8) * intCountMap.size()]; - BytesUtils.boolToBytes(nullCount != 0, bytes, 0); - if (nullCount != 0) { - BytesUtils.longToBytes(nullCount, bytes, 1); - } - BytesUtils.intToBytes(intCountMap.size(), bytes, offset); - offset += 4; - for (Map.Entry entry : intCountMap.entrySet()) { - BytesUtils.intToBytes(entry.getKey(), bytes, offset); - offset += 4; - BytesUtils.longToBytes(entry.getValue(), bytes, offset); - offset += 8; - } - break; - case FLOAT: - bytes = new byte[offset + 4 + (4 + 8) * floatCountMap.size()]; - BytesUtils.boolToBytes(nullCount != 0, bytes, 0); - if (nullCount != 0) { - BytesUtils.longToBytes(nullCount, bytes, 1); - } - BytesUtils.intToBytes(floatCountMap.size(), bytes, offset); - offset += 4; - for (Map.Entry entry : floatCountMap.entrySet()) { - BytesUtils.floatToBytes(entry.getKey(), bytes, offset); - offset += 4; - BytesUtils.longToBytes(entry.getValue(), bytes, offset); - offset += 8; - } - break; - case INT64: - case TIMESTAMP: - bytes = new byte[offset + 4 + (8 + 8) * longCountMap.size()]; - BytesUtils.boolToBytes(nullCount != 0, bytes, 0); - if (nullCount != 0) { - BytesUtils.longToBytes(nullCount, bytes, 1); - } - BytesUtils.intToBytes(longCountMap.size(), bytes, offset); - offset += 4; - for (Map.Entry entry : longCountMap.entrySet()) { - BytesUtils.longToBytes(entry.getKey(), bytes, offset); - offset += 8; - BytesUtils.longToBytes(entry.getValue(), bytes, offset); - offset += 8; - } - break; - case DOUBLE: - bytes = new byte[offset + 4 + (8 + 8) * doubleCountMap.size()]; - BytesUtils.boolToBytes(nullCount != 0, bytes, 0); - if (nullCount != 0) { - BytesUtils.longToBytes(nullCount, bytes, 1); - } - BytesUtils.intToBytes(doubleCountMap.size(), bytes, offset); - offset += 4; - for (Map.Entry entry : doubleCountMap.entrySet()) { - BytesUtils.doubleToBytes(entry.getKey(), bytes, offset); - offset += 8; - BytesUtils.longToBytes(entry.getValue(), bytes, offset); - offset += 8; - } - break; - case TEXT: - case STRING: - case BLOB: - bytes = - new byte - [offset - + 4 - + (8 + 4) * binaryCountMap.size() - + binaryCountMap.keySet().stream() - .mapToInt(key -> key.getValues().length) - .sum()]; - BytesUtils.boolToBytes(nullCount != 0, bytes, 0); - if (nullCount != 0) { - BytesUtils.longToBytes(nullCount, bytes, 1); - } - BytesUtils.intToBytes(binaryCountMap.size(), bytes, offset); - offset += 4; - for (Map.Entry entry : binaryCountMap.entrySet()) { - Binary binary = entry.getKey(); - serializeBinaryValue(binary, bytes, offset); - offset += (4 + binary.getLength()); - BytesUtils.longToBytes(entry.getValue(), bytes, offset); - offset += 8; - } - break; - default: - throw new UnsupportedOperationException( - String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); + int valueSize = + seriesDataType.isBinary() + ? countMap.keySet().stream().mapToInt(valueService::calcTypeSize).sum() + : countMap.size() * seriesDataType.getDataTypeSize(); + byte[] bytes = new byte[offset + Integer.BYTES + valueSize + Long.BYTES * countMap.size()]; + BytesUtils.boolToBytes(nullCount != 0, bytes, 0); + if (nullCount != 0) { + BytesUtils.longToBytes(nullCount, bytes, 1); + } + BytesUtils.intToBytes(countMap.size(), bytes, offset); + offset += Integer.BYTES; + for (Map.Entry entry : countMap.entrySet()) { + Object key = entry.getKey(); + offset += valueService.serialize(key, bytes, offset); + BytesUtils.longToBytes(entry.getValue(), bytes, offset); + offset += Long.BYTES; } - return bytes; } private void deserializeAndMergeCountMap(byte[] bytes) { int offset = 0; - if (bytesToBool(bytes, 0)) { - nullCount += bytesToLongFromOffset(bytes, Long.BYTES, 1); + if (BytesUtils.bytesToBool(bytes, 0)) { + nullCount += BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 1); offset += Long.BYTES; } offset++; int size = BytesUtils.bytesToInt(bytes, offset); - offset += 4; - - switch (seriesDataType) { - case BOOLEAN: - for (int i = 0; i < size; i++) { - boolean key = BytesUtils.bytesToBool(bytes, offset); - offset += 1; - long count = BytesUtils.bytesToLongFromOffset(bytes, 8, offset); - offset += 8; - booleanCountMap.compute(key, (k, v) -> v == null ? count : v + count); - } - break; - case INT32: - case DATE: - for (int i = 0; i < size; i++) { - int key = BytesUtils.bytesToInt(bytes, offset); - offset += 4; - long count = BytesUtils.bytesToLongFromOffset(bytes, 8, offset); - offset += 8; - intCountMap.compute(key, (k, v) -> v == null ? count : v + count); - } - break; - case FLOAT: - for (int i = 0; i < size; i++) { - float key = BytesUtils.bytesToFloat(bytes, offset); - offset += 4; - long count = BytesUtils.bytesToLongFromOffset(bytes, 8, offset); - offset += 8; - floatCountMap.compute(key, (k, v) -> v == null ? count : v + count); - } - break; - case INT64: - case TIMESTAMP: - for (int i = 0; i < size; i++) { - long key = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - offset += 8; - long count = BytesUtils.bytesToLongFromOffset(bytes, 8, offset); - offset += 8; - longCountMap.compute(key, (k, v) -> v == null ? count : v + count); - } - break; - case DOUBLE: - for (int i = 0; i < size; i++) { - double key = BytesUtils.bytesToDouble(bytes, offset); - offset += 8; - long count = BytesUtils.bytesToLongFromOffset(bytes, 8, offset); - offset += 8; - doubleCountMap.compute(key, (k, v) -> v == null ? count : v + count); - } - break; - case TEXT: - case STRING: - case BLOB: - for (int i = 0; i < size; i++) { - int length = BytesUtils.bytesToInt(bytes, offset); - offset += 4; - Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - offset += length; - long count = BytesUtils.bytesToLongFromOffset(bytes, 8, offset); - offset += 8; - binaryCountMap.compute(binaryVal, (k, v) -> v == null ? count : v + count); - } - break; - default: - throw new UnsupportedOperationException( - String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); - } - } - - private void addBooleanInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - booleanCountMap.compute(column.getBoolean(i), (k, v) -> v == null ? 1 : v + 1); - checkMapSize(booleanCountMap.size()); - } else { - nullCount++; - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - booleanCountMap.compute(column.getBoolean(position), (k, v) -> v == null ? 1 : v + 1); - checkMapSize(booleanCountMap.size()); - } else { - nullCount++; - } - } - } - } - - private void addIntInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - intCountMap.compute(column.getInt(i), (k, v) -> v == null ? 1 : v + 1); - checkMapSize(intCountMap.size()); - } else { - nullCount++; - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - intCountMap.compute(column.getInt(position), (k, v) -> v == null ? 1 : v + 1); - checkMapSize(intCountMap.size()); - } else { - nullCount++; - } - } - } - } - - private void addFloatInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - floatCountMap.compute(column.getFloat(i), (k, v) -> v == null ? 1 : v + 1); - checkMapSize(floatCountMap.size()); - } else { - nullCount++; - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - floatCountMap.compute(column.getFloat(position), (k, v) -> v == null ? 1 : v + 1); - checkMapSize(floatCountMap.size()); - } else { - nullCount++; - } - } - } - } - - private void addLongInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - longCountMap.compute(column.getLong(i), (k, v) -> v == null ? 1 : v + 1); - checkMapSize(longCountMap.size()); - } else { - nullCount++; - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - longCountMap.compute(column.getLong(position), (k, v) -> v == null ? 1 : v + 1); - checkMapSize(longCountMap.size()); - } else { - nullCount++; - } - } - } - } - - private void addDoubleInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - doubleCountMap.compute(column.getDouble(i), (k, v) -> v == null ? 1 : v + 1); - checkMapSize(doubleCountMap.size()); - } else { - nullCount++; - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - doubleCountMap.compute(column.getDouble(position), (k, v) -> v == null ? 1 : v + 1); - checkMapSize(doubleCountMap.size()); - } else { - nullCount++; - } - } - } - } - - private void addBinaryInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - binaryCountMap.compute(column.getBinary(i), (k, v) -> v == null ? 1 : v + 1); - checkMapSize(binaryCountMap.size()); - } else { - nullCount++; - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - binaryCountMap.compute(column.getBinary(position), (k, v) -> v == null ? 1 : v + 1); - checkMapSize(binaryCountMap.size()); - } else { - nullCount++; - } - } - } - } - - private void removeBooleanInput(Column column) { - for (int i = 0; i < column.getPositionCount(); i++) { - if (!column.isNull(i)) { - boolean key = column.getBoolean(i); - booleanCountMap.compute(key, (k, count) -> count - 1); - } else { - nullCount--; - } - } - } - - private void removeIntInput(Column column) { - for (int i = 0; i < column.getPositionCount(); i++) { - if (!column.isNull(i)) { - int key = column.getInt(i); - intCountMap.compute(key, (k, count) -> count - 1); - } else { - nullCount--; - } - } - } - - private void removeFloatInput(Column column) { - for (int i = 0; i < column.getPositionCount(); i++) { - if (!column.isNull(i)) { - float key = column.getFloat(i); - floatCountMap.compute(key, (k, count) -> count - 1); - } else { - nullCount--; - } - } - } - - private void removeLongInput(Column column) { - for (int i = 0; i < column.getPositionCount(); i++) { - if (!column.isNull(i)) { - long key = column.getLong(i); - longCountMap.compute(key, (k, count) -> count - 1); - } else { - nullCount--; - } - } - } - - private void removeDoubleInput(Column column) { - for (int i = 0; i < column.getPositionCount(); i++) { - if (!column.isNull(i)) { - double key = column.getDouble(i); - doubleCountMap.compute(key, (k, count) -> count - 1); - } else { - nullCount--; - } + offset += Integer.BYTES; + for (int i = 0; i < size; i++) { + Object key = valueService.deserialize(bytes, offset); + offset += valueService.calcTypeSize(key); + long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); + offset += Long.BYTES; + countMap.compute(key, (k, v) -> v == null ? count : v + count); } } - private void removeBinaryInput(Column column) { - for (int i = 0; i < column.getPositionCount(); i++) { - if (!column.isNull(i)) { - Binary key = column.getBinary(i); - binaryCountMap.compute(key, (k, count) -> count - 1); - } else { - nullCount--; - } + private void addValue(Column column, int position) { + if (column.isNull(position)) { + nullCount++; + return; } + countMap.compute( + valueService.getValue(column, position), (key, count) -> count == null ? 1L : count + 1); + checkMapSize(countMap.size()); } private void checkMapSize(int size) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableRegressionAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableRegressionAccumulator.java index 79f008eddc6a..c0532ed0aa9f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableRegressionAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableRegressionAccumulator.java @@ -26,7 +26,6 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; @@ -78,8 +77,8 @@ public void addInput(Column[] arguments, AggregationMask mask) { if (arguments[0].isNull(i) || arguments[1].isNull(i)) { continue; } - double y = getDoubleValue(arguments[0], i, yDataType); - double x = getDoubleValue(arguments[1], i, xDataType); + double y = getDoubleValue(arguments[0], i); + double x = getDoubleValue(arguments[1], i); update(x, y); } } else { @@ -89,31 +88,15 @@ public void addInput(Column[] arguments, AggregationMask mask) { if (arguments[0].isNull(position) || arguments[1].isNull(position)) { continue; } - double y = getDoubleValue(arguments[0], position, yDataType); - double x = getDoubleValue(arguments[1], position, xDataType); + double y = getDoubleValue(arguments[0], position); + double x = getDoubleValue(arguments[1], position); update(x, y); } } } - private double getDoubleValue(Column column, int position, TSDataType dataType) { - switch (dataType) { - case INT32: - case DATE: - return column.getInt(position); - case INT64: - case TIMESTAMP: - return column.getLong(position); - case FLOAT: - return column.getFloat(position); - case DOUBLE: - return column.getDouble(position); - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_REGRESSION_AGGREGATION_ARG_7BB08DA2, - dataType)); - } + private double getDoubleValue(Column column, int position) { + return column.getDouble(position); } private void update(double x, double y) { @@ -230,8 +213,8 @@ public void removeInput(Column[] arguments) { return; } - double otherY = getDoubleValue(arguments[0], 0, yDataType); - double otherX = getDoubleValue(arguments[1], 0, xDataType); + double otherY = getDoubleValue(arguments[0], 0); + double otherX = getDoubleValue(arguments[1], 0); if (count == 1) { reset(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java index 3b9c552d9546..c2aa7a4ca22f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java @@ -21,6 +21,7 @@ import org.apache.iotdb.calc.execution.aggregation.VarianceAccumulator; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -29,6 +30,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -41,6 +43,7 @@ public class TableVarianceAccumulator implements TableAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(TableVarianceAccumulator.class); private final TSDataType seriesDataType; + private final TypeServices.ColumnToDoubleConverter doubleValueConverter; private final VarianceAccumulator.VarianceType varianceType; private long count; @@ -50,6 +53,10 @@ public class TableVarianceAccumulator implements TableAccumulator { public TableVarianceAccumulator( TSDataType seriesDataType, VarianceAccumulator.VarianceType varianceType) { this.seriesDataType = seriesDataType; + this.doubleValueConverter = + TypeServices.NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(Type.fromTsDataType(seriesDataType)) + .create(this::unsupportedDataTypeException); this.varianceType = varianceType; } @@ -65,61 +72,18 @@ public TableAccumulator copy() { @Override public void addInput(Column[] arguments, AggregationMask mask) { - switch (seriesDataType) { - case INT32: - addIntInput(arguments[0], mask); - return; - case INT64: - addLongInput(arguments[0], mask); - return; - case FLOAT: - addFloatInput(arguments[0], mask); - return; - case DOUBLE: - addDoubleInput(arguments[0], mask); - return; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case STRING: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_VARIANCE_AGGREGATION_ARG_C641D425, - seriesDataType)); - } + checkInputDataType(); + updateStateByAdd(arguments[0], mask); } @Override public void removeInput(Column[] arguments) { - switch (seriesDataType) { - case INT32: - removeIntInput(arguments[0]); - return; - case INT64: - removeLongInput(arguments[0]); - return; - case FLOAT: - removeFloatInput(arguments[0]); - return; - case DOUBLE: - removeDoubleInput(arguments[0]); - return; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case STRING: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_VARIANCE_AGGREGATION_ARG_C641D425, - seriesDataType)); + checkInputDataType(); + Column column = arguments[0]; + for (int i = 0; i < column.getPositionCount(); i++) { + if (!column.isNull(i)) { + updateStateByRemove(doubleValueConverter.convert(column, i)); + } } } @@ -227,106 +191,7 @@ public boolean removable() { return true; } - private void addIntInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (column.isNull(i)) { - continue; - } - - int value = column.getInt(i); - count++; - double delta = value - mean; - mean += delta / count; - m2 += delta * (value - mean); - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (column.isNull(position)) { - continue; - } - - int value = column.getInt(position); - count++; - double delta = value - mean; - mean += delta / count; - m2 += delta * (value - mean); - } - } - } - - private void addLongInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (column.isNull(i)) { - continue; - } - - long value = column.getLong(i); - count++; - double delta = value - mean; - mean += delta / count; - m2 += delta * (value - mean); - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (column.isNull(position)) { - continue; - } - - long value = column.getLong(position); - count++; - double delta = value - mean; - mean += delta / count; - m2 += delta * (value - mean); - } - } - } - - private void addFloatInput(Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (column.isNull(i)) { - continue; - } - - float value = column.getFloat(i); - count++; - double delta = value - mean; - mean += delta / count; - m2 += delta * (value - mean); - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (column.isNull(position)) { - continue; - } - - float value = column.getFloat(position); - count++; - double delta = value - mean; - mean += delta / count; - m2 += delta * (value - mean); - } - } - } - - private void addDoubleInput(Column column, AggregationMask mask) { + private void updateStateByAdd(Column column, AggregationMask mask) { int positionCount = mask.getSelectedPositionCount(); if (mask.isSelectAll()) { @@ -335,7 +200,7 @@ private void addDoubleInput(Column column, AggregationMask mask) { continue; } - double value = column.getDouble(i); + double value = doubleValueConverter.convert(column, i); count++; double delta = value - mean; mean += delta / count; @@ -350,7 +215,7 @@ private void addDoubleInput(Column column, AggregationMask mask) { continue; } - double value = column.getDouble(position); + double value = doubleValueConverter.convert(column, position); count++; double delta = value - mean; mean += delta / count; @@ -359,50 +224,6 @@ private void addDoubleInput(Column column, AggregationMask mask) { } } - private void removeIntInput(Column column) { - for (int i = 0; i < column.getPositionCount(); i++) { - if (column.isNull(i)) { - continue; - } - - int value = column.getInt(i); - updateStateByRemove(value); - } - } - - private void removeLongInput(Column column) { - for (int i = 0; i < column.getPositionCount(); i++) { - if (column.isNull(i)) { - continue; - } - - long value = column.getLong(i); - updateStateByRemove(value); - } - } - - private void removeFloatInput(Column column) { - for (int i = 0; i < column.getPositionCount(); i++) { - if (column.isNull(i)) { - continue; - } - - float value = column.getFloat(i); - updateStateByRemove(value); - } - } - - private void removeDoubleInput(Column column) { - for (int i = 0; i < column.getPositionCount(); i++) { - if (column.isNull(i)) { - continue; - } - - double value = column.getDouble(i); - updateStateByRemove(value); - } - } - private void updateStateByRemove(double value) { long newCount = count - 1; double newMean = (count * mean - value) / newCount; @@ -412,4 +233,15 @@ private void updateStateByRemove(double value) { count = newCount; mean = newMean; } + + private void checkInputDataType() { + if (!seriesDataType.isNumeric()) { + throw unsupportedDataTypeException(); + } + } + + private UnSupportedDataTypeException unsupportedDataTypeException() { + return new UnSupportedDataTypeException( + String.format(CalcMessages.UNSUPPORTED_DATA_TYPE_IN_VARIANCE_AGGREGATION, seriesDataType)); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/Utils.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/Utils.java index 611545267165..8bd550bbdd54 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/Utils.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/Utils.java @@ -19,59 +19,22 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; -import org.apache.iotdb.calc.i18n.CalcMessages; - import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import static org.apache.tsfile.enums.TSDataType.BLOB; import static org.apache.tsfile.enums.TSDataType.STRING; import static org.apache.tsfile.enums.TSDataType.TEXT; public class Utils { - public static final String UNSUPPORTED_TYPE_MESSAGE = "Unsupported data type : %s"; - private Utils() {} public static void serializeValue( TSDataType dataType, TsPrimitiveType value, byte[] valueBytes, int offset) { - switch (dataType) { - case INT32: - case DATE: - BytesUtils.intToBytes(value.getInt(), valueBytes, offset); - break; - case INT64: - case TIMESTAMP: - BytesUtils.longToBytes(value.getLong(), valueBytes, offset); - break; - case FLOAT: - BytesUtils.floatToBytes(value.getFloat(), valueBytes, offset); - break; - case DOUBLE: - BytesUtils.doubleToBytes(value.getDouble(), valueBytes, offset); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - BytesUtils.intToBytes(value.getBinary().getValues().length, valueBytes, offset); - offset += 4; - System.arraycopy( - value.getBinary().getValues(), - 0, - valueBytes, - offset, - value.getBinary().getValues().length); - break; - case BOOLEAN: - BytesUtils.boolToBytes(value.getBoolean(), valueBytes, offset); - break; - default: - throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, dataType)); - } + Type.fromTsDataType(dataType).toBytes(value, valueBytes, offset); } public static void serializeBinaryValue(Binary binary, byte[] valueBytes, int offset) { @@ -111,26 +74,7 @@ public static byte[] serializeTimeValueWithNull( } public static int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { - switch (dataType) { - case BOOLEAN: - return 1; - case INT32: - case DATE: - case FLOAT: - return 4; - case INT64: - case TIMESTAMP: - case DOUBLE: - return 8; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - return 4 + value.getBinary().getValues().length; - default: - throw new UnSupportedDataTypeException( - String.format(CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_7D59F7B2, dataType)); - } + return Type.fromTsDataType(dataType).calcTypeSize(value); } public static boolean isBinaryType(TSDataType dataType) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/AbstractGroupedApproxPercentileAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/AbstractGroupedApproxPercentileAccumulator.java index 075650e6115c..fd637beb1f5c 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/AbstractGroupedApproxPercentileAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/AbstractGroupedApproxPercentileAccumulator.java @@ -23,14 +23,15 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.TDigest; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.TDigestBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; @@ -39,11 +40,19 @@ public abstract class AbstractGroupedApproxPercentileAccumulator implements Grou private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedApproxPercentileAccumulator.class); protected final TSDataType seriesDataType; + private final TypeServices.NumericResultWriter resultWriter; + private final TypeServices.TriConsumer inputAdder; protected double percentage; protected final TDigestBigArray array = new TDigestBigArray(); AbstractGroupedApproxPercentileAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; + this.resultWriter = + TypeServices.NUMERIC_RESULT_WRITER_SERVICE.call(Type.fromTsDataType(seriesDataType)); + this.inputAdder = + TypeServices.GROUPED_APPROX_PERCENTILE_INPUT_SERVICE + .call(Type.fromTsDataType(seriesDataType)) + .apply(this); } @Override @@ -65,31 +74,10 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { } else { throw new IllegalArgumentException( String.format( - CalcMessages.EXCEPTION_APPROX_PERCENTILE_REQUIRES_2_3_ARGUMENTS_BUT_GOT_ARG_D78590AA, - arguments.length)); + CalcMessages.APPROX_PERCENTILE_REQUIRES_TWO_OR_THREE_ARGUMENTS, arguments.length)); } - switch (seriesDataType) { - case INT32: - addIntInput(groupIds, arguments, mask); - break; - case INT64: - case TIMESTAMP: - addLongInput(groupIds, arguments, mask); - break; - case FLOAT: - addFloatInput(groupIds, arguments, mask); - break; - case DOUBLE: - addDoubleInput(groupIds, arguments, mask); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages - .EXCEPTION_UNSUPPORTED_DATA_TYPE_APPROX_PERCENTILE_AGGREGATION_ARG_CFEC0431, - seriesDataType)); - } + inputAdder.accept(groupIds, arguments, mask); } @Override @@ -124,27 +112,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (seriesDataType) { - case INT32: - columnBuilder.writeInt((int) result); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong((long) result); - break; - case FLOAT: - columnBuilder.writeFloat((float) result); - break; - case DOUBLE: - columnBuilder.writeDouble(result); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages - .EXCEPTION_UNSUPPORTED_DATA_TYPE_APPROX_PERCENTILE_AGGREGATION_ARG_CFEC0431, - seriesDataType)); - } + resultWriter.write(columnBuilder, result); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java index 5a2d38af3584..a02f39d85e66 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java @@ -23,14 +23,14 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.HyperLogLog; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.HyperLogLogStateFactory; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.HyperLogLogBigArray; -import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.write.UnSupportedDataTypeException; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.HyperLogLog.DEFAULT_STANDARD_ERROR; @@ -38,12 +38,15 @@ public class GroupedApproxCountDistinctAccumulator implements GroupedAccumulator private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedApproxCountDistinctAccumulator.class); private final TSDataType seriesDataType; + private final TypeServices.HyperLogLogColumnAdder valueAdder; private final HyperLogLogStateFactory.GroupedHyperLogLogState state = HyperLogLogStateFactory.createGroupedState(); public GroupedApproxCountDistinctAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; + this.valueAdder = + TypeServices.HYPER_LOG_LOG_COLUMN_ADDER_SERVICE.call(Type.fromTsDataType(seriesDataType)); } @Override @@ -63,36 +66,24 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { arguments.length == 1 ? DEFAULT_STANDARD_ERROR : arguments[1].getDouble(0); HyperLogLogBigArray hlls = getOrCreateHyperLogLog(state); - switch (seriesDataType) { - case BOOLEAN: - addBooleanInput(groupIds, arguments[0], mask, hlls, maxStandardError); - return; - case INT32: - case DATE: - addIntInput(groupIds, arguments[0], mask, hlls, maxStandardError); - return; - case INT64: - case TIMESTAMP: - addLongInput(groupIds, arguments[0], mask, hlls, maxStandardError); - return; - case FLOAT: - addFloatInput(groupIds, arguments[0], mask, hlls, maxStandardError); - return; - case DOUBLE: - addDoubleInput(groupIds, arguments[0], mask, hlls, maxStandardError); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(groupIds, arguments[0], mask, hlls, maxStandardError); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages - .EXCEPTION_UNSUPPORTED_DATA_TYPE_APPROX_COUNT_DISTINCT_AGGREGATION_ARG_58F0391E, - seriesDataType)); + Column valueColumn = arguments[0]; + int positionCount = mask.getPositionCount(); + if (mask.isSelectAll()) { + for (int i = 0; i < positionCount; i++) { + HyperLogLog hll = hlls.get(groupIds[i], maxStandardError); + if (!valueColumn.isNull(i)) { + valueAdder.add(hll, valueColumn, i); + } + } + } else { + int[] selectedPositions = mask.getSelectedPositions(); + for (int i = 0; i < positionCount; i++) { + int position = selectedPositions[i]; + HyperLogLog hll = hlls.get(groupIds[position], maxStandardError); + if (!valueColumn.isNull(position)) { + valueAdder.add(hll, valueColumn, position); + } + } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java index e241f2c1d7b7..bce23d17e279 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java @@ -22,6 +22,7 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.DoubleBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -29,6 +30,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -42,11 +44,22 @@ public class GroupedAvgAccumulator implements GroupedAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedAvgAccumulator.class); private final TSDataType argumentDataType; + private final TypeServices.ColumnToDoubleConverter valueConverter; private final LongBigArray countValues = new LongBigArray(); private final DoubleBigArray sumValues = new DoubleBigArray(); public GroupedAvgAccumulator(TSDataType argumentDataType) { this.argumentDataType = argumentDataType; + Type type = Type.fromTsDataType(argumentDataType); + this.valueConverter = + TypeServices.AGGREGATION_NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(type) + .create( + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_AGGREGATION_AVG, + argumentDataType))); } @Override @@ -62,35 +75,8 @@ public void setGroupCount(long groupCount) { @Override public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - checkArgument( - arguments.length == 1, - CalcMessages.EXCEPTION_ARGUMENT_OF_AVG_SHOULD_BE_ONE_COLUMN_82162B82); - switch (argumentDataType) { - case INT32: - addIntInput(groupIds, arguments[0], mask); - return; - case INT64: - addLongInput(groupIds, arguments[0], mask); - return; - case FLOAT: - addFloatInput(groupIds, arguments[0], mask); - return; - case DOUBLE: - addDoubleInput(groupIds, arguments[0], mask); - return; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_AVG_AGGREGATION_ARG_66BF3128, - argumentDataType)); - } + checkArgument(arguments.length == 1, "argument of AVG should be one column"); + addInput(groupIds, arguments[0], mask); } @Override @@ -134,89 +120,14 @@ private byte[] serializeState(int groupId) { return bytes; } - private void addIntInput(int[] groupIds, Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - countValues.increment(groupIds[i]); - sumValues.add(groupIds[i], column.getInt(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - int groupId; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - groupId = groupIds[position]; - if (!column.isNull(position)) { - countValues.increment(groupId); - sumValues.add(groupId, column.getInt(position)); - } - } - } - } - - private void addLongInput(int[] groupIds, Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - countValues.increment(groupIds[i]); - sumValues.add(groupIds[i], column.getLong(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - int groupId; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - groupId = groupIds[position]; - if (!column.isNull(position)) { - countValues.increment(groupId); - sumValues.add(groupId, column.getLong(position)); - } - } - } - } - - private void addFloatInput(int[] groupIds, Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - countValues.increment(groupIds[i]); - sumValues.add(groupIds[i], column.getFloat(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - int groupId; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - groupId = groupIds[position]; - if (!column.isNull(position)) { - countValues.increment(groupId); - sumValues.add(groupId, column.getFloat(position)); - } - } - } - } - - private void addDoubleInput(int[] groupIds, Column column, AggregationMask mask) { + private void addInput(int[] groupIds, Column column, AggregationMask mask) { int positionCount = mask.getSelectedPositionCount(); if (mask.isSelectAll()) { for (int i = 0; i < positionCount; i++) { if (!column.isNull(i)) { countValues.increment(groupIds[i]); - sumValues.add(groupIds[i], column.getDouble(i)); + sumValues.add(groupIds[i], valueConverter.convert(column, i)); } } } else { @@ -228,7 +139,7 @@ private void addDoubleInput(int[] groupIds, Column column, AggregationMask mask) groupId = groupIds[position]; if (!column.isNull(position)) { countValues.increment(groupId); - sumValues.add(groupId, column.getDouble(position)); + sumValues.add(groupId, valueConverter.convert(column, position)); } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedCentralMomentAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedCentralMomentAccumulator.java index 2571c33a8249..73869dcd52be 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedCentralMomentAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedCentralMomentAccumulator.java @@ -24,6 +24,7 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.DoubleBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -31,6 +32,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -46,7 +48,7 @@ public class GroupedCentralMomentAccumulator implements GroupedAccumulator { private static final int INTERMEDIATE_SIZE = Long.BYTES + 4 * Double.BYTES; private static final double EPSILON = 1e-12; - private final TSDataType seriesDataType; + private final TypeServices.ColumnToDoubleConverter doubleValueConverter; private final CentralMomentAccumulator.MomentType momentType; private final LongBigArray counts = new LongBigArray(); @@ -57,7 +59,15 @@ public class GroupedCentralMomentAccumulator implements GroupedAccumulator { public GroupedCentralMomentAccumulator( TSDataType seriesDataType, CentralMomentAccumulator.MomentType momentType) { - this.seriesDataType = seriesDataType; + this.doubleValueConverter = + TypeServices.NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(Type.fromTsDataType(seriesDataType)) + .create( + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_CENTRAL_MOMENT_AGGREGATION, + seriesDataType))); this.momentType = momentType; } @@ -86,7 +96,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { if (mask.isSelectAll()) { for (int i = 0; i < positionCount; i++) { if (!arguments[0].isNull(i)) { - update(groupIds[i], getDoubleValue(arguments[0], i)); + update(groupIds[i], doubleValueConverter.convert(arguments[0], i)); } } } else { @@ -94,32 +104,12 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { for (int i = 0; i < positionCount; i++) { int position = selectedPositions[i]; if (!arguments[0].isNull(position)) { - update(groupIds[position], getDoubleValue(arguments[0], position)); + update(groupIds[position], doubleValueConverter.convert(arguments[0], position)); } } } } - private double getDoubleValue(Column column, int position) { - switch (seriesDataType) { - case INT32: - case DATE: - return column.getInt(position); - case INT64: - case TIMESTAMP: - return column.getLong(position); - case FLOAT: - return column.getFloat(position); - case DOUBLE: - return column.getDouble(position); - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_CENTRALMOMENT_AGGREGATION_ARG_74B9A3A8, - seriesDataType)); - } - } - private void update(int groupId, double value) { long n1 = counts.get(groupId); long n = n1 + 1; diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedCorrelationAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedCorrelationAccumulator.java index fd73c66db7b2..31b8d66621c0 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedCorrelationAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedCorrelationAccumulator.java @@ -32,7 +32,6 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; @@ -88,8 +87,8 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { if (arguments[0].isNull(i) || arguments[1].isNull(i)) { continue; } - double x = getDoubleValue(arguments[0], i, xDataType); - double y = getDoubleValue(arguments[1], i, yDataType); + double x = getDoubleValue(arguments[0], i); + double y = getDoubleValue(arguments[1], i); update(groupIds[i], x, y); } } else { @@ -99,31 +98,15 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { if (arguments[0].isNull(position) || arguments[1].isNull(position)) { continue; } - double x = getDoubleValue(arguments[0], position, xDataType); - double y = getDoubleValue(arguments[1], position, yDataType); + double x = getDoubleValue(arguments[0], position); + double y = getDoubleValue(arguments[1], position); update(groupIds[position], x, y); } } } - private double getDoubleValue(Column column, int position, TSDataType dataType) { - switch (dataType) { - case INT32: - case DATE: - return column.getInt(position); - case INT64: - case TIMESTAMP: - return column.getLong(position); - case FLOAT: - return column.getFloat(position); - case DOUBLE: - return column.getDouble(position); - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_CORRELATION_AGGREGATION_ARG_2FF29597, - dataType)); - } + private double getDoubleValue(Column column, int position) { + return column.getDouble(position); } private void update(int groupId, double x, double y) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedCovarianceAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedCovarianceAccumulator.java index 5b615133d015..b6838db0c82f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedCovarianceAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedCovarianceAccumulator.java @@ -33,7 +33,6 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; @@ -85,8 +84,8 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { if (arguments[0].isNull(i) || arguments[1].isNull(i)) { continue; } - double x = getDoubleValue(arguments[0], i, xDataType); - double y = getDoubleValue(arguments[1], i, yDataType); + double x = getDoubleValue(arguments[0], i); + double y = getDoubleValue(arguments[1], i); update(groupIds[i], x, y); } } else { @@ -96,31 +95,15 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { if (arguments[0].isNull(position) || arguments[1].isNull(position)) { continue; } - double x = getDoubleValue(arguments[0], position, xDataType); - double y = getDoubleValue(arguments[1], position, yDataType); + double x = getDoubleValue(arguments[0], position); + double y = getDoubleValue(arguments[1], position); update(groupIds[position], x, y); } } } - private double getDoubleValue(Column column, int position, TSDataType dataType) { - switch (dataType) { - case INT32: - case DATE: - return column.getInt(position); - case INT64: - case TIMESTAMP: - return column.getLong(position); - case FLOAT: - return column.getFloat(position); - case DOUBLE: - return column.getDouble(position); - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_COVARIANCE_AGGREGATION_ARG_643BEDD0, - dataType)); - } + private double getDoubleValue(Column column, int position) { + return column.getDouble(position); } private void update(int groupId, double x, double y) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java index 1a871639af40..0e2a746e6493 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java @@ -20,20 +20,27 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BinaryBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BooleanBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.DoubleBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.FloatBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.UnSupportedDataTypeException; -public class GroupedExtremeAccumulator implements GroupedAccumulator { +public class GroupedExtremeAccumulator + implements GroupedAccumulator, + TypeServices.GroupedValueAccessor, + TypeServices.GroupedInputConsumer { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedExtremeAccumulator.class); private final TSDataType seriesDataType; @@ -44,169 +51,38 @@ public class GroupedExtremeAccumulator implements GroupedAccumulator { private IntBigArray intValues; private FloatBigArray floatValues; private DoubleBigArray doubleValues; + private final TypeServices.GroupedValueService valueService; + private final TypeServices.GroupedInputReader inputReader; public GroupedExtremeAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - switch (seriesDataType) { - case INT32: - intValues = new IntBigArray(); - return; - case INT64: - longValues = new LongBigArray(); - return; - case FLOAT: - floatValues = new FloatBigArray(); - return; - case DOUBLE: - doubleValues = new DoubleBigArray(); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_EXTREME_AGGREGATION_ARG_AF4DA98F, - seriesDataType)); - } + Type type = Type.fromTsDataType(seriesDataType); + this.valueService = TypeServices.GROUPED_EXTREME_VALUE_SERVICE.call(type); + this.inputReader = TypeServices.GROUPED_EXTREME_INPUT_READER_SERVICE.call(type); + valueService.initialize(this); } @Override public long getEstimatedSize() { - long valuesSize = 0; - switch (seriesDataType) { - case INT32: - case DATE: - valuesSize += intValues.sizeOf(); - break; - case INT64: - case TIMESTAMP: - valuesSize += longValues.sizeOf(); - break; - case FLOAT: - valuesSize += floatValues.sizeOf(); - break; - case DOUBLE: - valuesSize += doubleValues.sizeOf(); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - break; - case BOOLEAN: - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_EXTREME_AGGREGATION_ARG_276BE220, - seriesDataType)); - } - - return INSTANCE_SIZE + valuesSize; + return INSTANCE_SIZE + valueService.sizeOf(this); } @Override public void setGroupCount(long groupCount) { inits.ensureCapacity(groupCount); - switch (seriesDataType) { - case INT32: - case DATE: - intValues.ensureCapacity(groupCount); - return; - case INT64: - case TIMESTAMP: - longValues.ensureCapacity(groupCount); - return; - case FLOAT: - floatValues.ensureCapacity(groupCount); - return; - case DOUBLE: - doubleValues.ensureCapacity(groupCount); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_EXTREME_AGGREGATION_ARG_276BE220, - seriesDataType)); - } + valueService.ensureCapacity(this, groupCount); } @Override public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - switch (seriesDataType) { - case INT32: - addIntInput(groupIds, arguments[0], mask); - return; - case INT64: - addLongInput(groupIds, arguments[0], mask); - return; - case FLOAT: - addFloatInput(groupIds, arguments[0], mask); - return; - case DOUBLE: - addDoubleInput(groupIds, arguments[0], mask); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_EXTREME_AGGREGATION_ARG_AF4DA98F, - seriesDataType)); - } + inputReader.addInput(groupIds, arguments, mask, this); } @Override public void addIntermediate(int[] groupIds, Column argument) { - for (int i = 0; i < groupIds.length; i++) { - if (argument.isNull(i)) { - continue; - } - - switch (seriesDataType) { - case INT32: - updateIntValue(groupIds[i], argument.getInt(i)); - break; - case INT64: - updateLongValue(groupIds[i], argument.getLong(i)); - break; - case FLOAT: - updateFloatValue(groupIds[i], argument.getFloat(i)); - break; - case DOUBLE: - updateDoubleValue(groupIds[i], argument.getDouble(i)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_EXTREME_AGGREGATION_ARG_276BE220, - seriesDataType)); - } - } + inputReader.addIntermediate(groupIds, argument, this); } @Override @@ -214,32 +90,7 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { if (!inits.get(groupId)) { columnBuilder.appendNull(); } else { - switch (seriesDataType) { - case INT32: - columnBuilder.writeInt(intValues.get(groupId)); - break; - case INT64: - columnBuilder.writeLong(longValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(floatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(doubleValues.get(groupId)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_EXTREME_AGGREGATION_ARG_276BE220, - seriesDataType)); - } + valueService.write(this, groupId, columnBuilder); } } @@ -248,32 +99,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { if (!inits.get(groupId)) { columnBuilder.appendNull(); } else { - switch (seriesDataType) { - case INT32: - columnBuilder.writeInt(intValues.get(groupId)); - break; - case INT64: - columnBuilder.writeLong(longValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(floatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(doubleValues.get(groupId)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_EXTREME_AGGREGATION_ARG_276BE220, - seriesDataType)); - } + valueService.write(this, groupId, columnBuilder); } } @@ -283,32 +109,7 @@ public void prepareFinal() {} @Override public void reset() { inits.reset(); - switch (seriesDataType) { - case INT32: - intValues.reset(); - return; - case INT64: - longValues.reset(); - return; - case FLOAT: - floatValues.reset(); - return; - case DOUBLE: - doubleValues.reset(); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_EXTREME_AGGREGATION_ARG_276BE220, - seriesDataType)); - } + valueService.reset(this); } private void addIntInput(int[] groupIds, Column valueColumn, AggregationMask mask) { @@ -456,4 +257,100 @@ private int compareExtreme(double left, double right) { int absComparison = Double.compare(Math.abs(left), Math.abs(right)); return absComparison == 0 ? Double.compare(left, right) : absComparison; } + + @Override + public RuntimeException unsupportedException() { + return new UnSupportedDataTypeException( + String.format(CalcMessages.UNSUPPORTED_DATA_TYPE_IN_EXTREME_AGGREGATION, seriesDataType)); + } + + @Override + public void initializeIntValues() { + intValues = new IntBigArray(); + } + + @Override + public void initializeLongValues() { + longValues = new LongBigArray(); + } + + @Override + public void initializeFloatValues() { + floatValues = new FloatBigArray(); + } + + @Override + public void initializeDoubleValues() { + doubleValues = new DoubleBigArray(); + } + + @Override + public void initializeBinaryValues() { + throw unsupportedException(); + } + + @Override + public void initializeBooleanValues() { + throw unsupportedException(); + } + + @Override + public IntBigArray getIntValues() { + return intValues; + } + + @Override + public LongBigArray getLongValues() { + return longValues; + } + + @Override + public FloatBigArray getFloatValues() { + return floatValues; + } + + @Override + public DoubleBigArray getDoubleValues() { + return doubleValues; + } + + @Override + public BinaryBigArray getBinaryValues() { + throw unsupportedException(); + } + + @Override + public BooleanBigArray getBooleanValues() { + throw unsupportedException(); + } + + @Override + public void updateInt(int groupId, int value) { + updateIntValue(groupId, value); + } + + @Override + public void updateLong(int groupId, long value) { + updateLongValue(groupId, value); + } + + @Override + public void updateFloat(int groupId, float value) { + updateFloatValue(groupId, value); + } + + @Override + public void updateDouble(int groupId, double value) { + updateDoubleValue(groupId, value); + } + + @Override + public void updateBinary(int groupId, Binary value) { + throw unsupportedException(); + } + + @Override + public void updateBoolean(int groupId, boolean value) { + throw unsupportedException(); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java index 640d6a5a4739..e22ab23f1d68 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java @@ -27,6 +27,7 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -34,6 +35,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -41,15 +43,19 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.tsfile.utils.BytesUtils.boolToBytes; -import static org.apache.tsfile.utils.BytesUtils.doubleToBytes; -import static org.apache.tsfile.utils.BytesUtils.floatToBytes; -import static org.apache.tsfile.utils.BytesUtils.intToBytes; import static org.apache.tsfile.utils.BytesUtils.longToBytes; -public class GroupedFirstAccumulator implements GroupedAccumulator { +public class GroupedFirstAccumulator + implements GroupedAccumulator, + TypeServices.GroupedValueAccessor, + TypeServices.GroupedTimeValueConsumer { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedFirstAccumulator.class); private final TSDataType seriesDataType; + private final TypeServices.GroupedValueService valueService; + private final TypeServices.GroupedValueSerializer valueSerializer; + private final TypeServices.GroupedTimeInputReader inputReader; + private final TypeServices.GroupedTimeValueDeserializer valueDeserializer; private final LongBigArray minTimes = new LongBigArray(Long.MAX_VALUE); private final BooleanBigArray inits = new BooleanBigArray(); private final BooleanBigArray initNullTimeValues = new BooleanBigArray(); @@ -63,77 +69,21 @@ public class GroupedFirstAccumulator implements GroupedAccumulator { public GroupedFirstAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - switch (seriesDataType) { - case INT32: - case DATE: - intValues = new IntBigArray(); - return; - case INT64: - case TIMESTAMP: - longValues = new LongBigArray(); - return; - case FLOAT: - floatValues = new FloatBigArray(); - return; - case DOUBLE: - doubleValues = new DoubleBigArray(); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - binaryValues = new BinaryBigArray(); - return; - case BOOLEAN: - booleanValues = new BooleanBigArray(); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_B2D27BB9, - seriesDataType)); - } + Type type = Type.fromTsDataType(seriesDataType); + this.valueService = TypeServices.GROUPED_VALUE_SERVICE.call(type); + this.valueSerializer = TypeServices.GROUPED_VALUE_SERIALIZER_SERVICE.call(type); + this.inputReader = TypeServices.GROUPED_TIME_INPUT_READER_SERVICE.call(type); + this.valueDeserializer = TypeServices.GROUPED_TIME_VALUE_DESERIALIZER_SERVICE.call(type); + valueService.initialize(this); } @Override public long getEstimatedSize() { - long valuesSize = 0; - switch (seriesDataType) { - case INT32: - case DATE: - valuesSize += intValues.sizeOf(); - break; - case INT64: - case TIMESTAMP: - valuesSize += longValues.sizeOf(); - break; - case FLOAT: - valuesSize += floatValues.sizeOf(); - break; - case DOUBLE: - valuesSize += doubleValues.sizeOf(); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - valuesSize += binaryValues.sizeOf(); - break; - case BOOLEAN: - valuesSize += booleanValues.sizeOf(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_B2D27BB9, - seriesDataType)); - } - return INSTANCE_SIZE + inits.sizeOf() + minTimes.sizeOf() + initNullTimeValues.sizeOf() - + valuesSize; + + valueService.sizeOf(this); } @Override @@ -141,71 +91,13 @@ public void setGroupCount(long groupCount) { minTimes.ensureCapacity(groupCount); inits.ensureCapacity(groupCount); initNullTimeValues.ensureCapacity(groupCount); - switch (seriesDataType) { - case INT32: - case DATE: - intValues.ensureCapacity(groupCount); - return; - case INT64: - case TIMESTAMP: - longValues.ensureCapacity(groupCount); - return; - case FLOAT: - floatValues.ensureCapacity(groupCount); - return; - case DOUBLE: - doubleValues.ensureCapacity(groupCount); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - binaryValues.ensureCapacity(groupCount); - return; - case BOOLEAN: - booleanValues.ensureCapacity(groupCount); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_B2D27BB9, - seriesDataType)); - } + valueService.ensureCapacity(this, groupCount); } @Override public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { // arguments[0] is value column, arguments[1] is time column - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(groupIds, arguments[0], arguments[1], mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(groupIds, arguments[0], arguments[1], mask); - return; - case FLOAT: - addFloatInput(groupIds, arguments[0], arguments[1], mask); - return; - case DOUBLE: - addDoubleInput(groupIds, arguments[0], arguments[1], mask); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(groupIds, arguments[0], arguments[1], mask); - return; - case BOOLEAN: - addBooleanInput(groupIds, arguments[0], arguments[1], mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_B2D27BB9, - seriesDataType)); - } + inputReader.addInput(groupIds, arguments, mask, this); } @Override @@ -228,69 +120,7 @@ public void addIntermediate(int[] groupIds, Column argument) { boolean isOrderTimeNull = BytesUtils.bytesToBool(bytes, offset); offset += 1; int groupId = groupIds[i]; - - switch (seriesDataType) { - case INT32: - case DATE: - int intVal = BytesUtils.bytesToInt(bytes, offset); - if (!isOrderTimeNull) { - updateIntValue(groupId, intVal, time); - } else { - updateIntNullTimeValue(groupId, intVal); - } - break; - case INT64: - case TIMESTAMP: - long longVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - if (!isOrderTimeNull) { - updateLongValue(groupId, longVal, time); - } else { - updateLongNullTimeValue(groupId, longVal); - } - break; - case FLOAT: - float floatVal = BytesUtils.bytesToFloat(bytes, offset); - if (!isOrderTimeNull) { - updateFloatValue(groupId, floatVal, time); - } else { - updateFloatNullTimeValue(groupId, floatVal); - } - break; - case DOUBLE: - double doubleVal = BytesUtils.bytesToDouble(bytes, offset); - if (!isOrderTimeNull) { - updateDoubleValue(groupId, doubleVal, time); - } else { - updateDoubleNullTimeValue(groupId, doubleVal); - } - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - if (!isOrderTimeNull) { - updateBinaryValue(groupId, binaryVal, time); - } else { - updateBinaryNullTimeValue(groupId, binaryVal); - } - break; - case BOOLEAN: - boolean boolVal = BytesUtils.bytesToBool(bytes, offset); - if (!isOrderTimeNull) { - updateBooleanValue(groupId, boolVal, time); - } else { - updateBooleanNullTimeValue(groupId, boolVal); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_B2D27BB9, - seriesDataType)); - } + valueDeserializer.deserialize(bytes, offset, groupId, time, isOrderTimeNull, this); } } @@ -314,36 +144,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(intValues.get(groupId)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(longValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(floatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(doubleValues.get(groupId)); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - columnBuilder.writeBinary(binaryValues.get(groupId)); - break; - case BOOLEAN: - columnBuilder.writeBoolean(booleanValues.get(groupId)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_B2D27BB9, - seriesDataType)); - } + valueService.write(this, groupId, columnBuilder); } @Override @@ -354,100 +155,17 @@ public void reset() { minTimes.reset(); inits.reset(); initNullTimeValues.reset(); - switch (seriesDataType) { - case INT32: - case DATE: - intValues.reset(); - return; - case INT64: - case TIMESTAMP: - longValues.reset(); - return; - case FLOAT: - floatValues.reset(); - return; - case DOUBLE: - doubleValues.reset(); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - binaryValues.reset(); - return; - case BOOLEAN: - booleanValues.reset(); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_B2D27BB9, - seriesDataType)); - } + valueService.reset(this); } private byte[] serializeTimeWithValue(int groupId) { - byte[] bytes; - int length = Long.BYTES; + int length = Long.BYTES + 1 + valueSerializer.calcTypeSize(this, groupId); boolean isOrderTimeNull = !inits.get(groupId); - length += 1; - - switch (seriesDataType) { - case INT32: - case DATE: - length += Integer.BYTES; - bytes = new byte[length]; - longToBytes(minTimes.get(groupId), bytes, 0); - boolToBytes(isOrderTimeNull, bytes, Long.BYTES); - intToBytes(intValues.get(groupId), bytes, Long.BYTES + 1); - return bytes; - case INT64: - case TIMESTAMP: - length += Long.BYTES; - bytes = new byte[length]; - longToBytes(minTimes.get(groupId), bytes, 0); - boolToBytes(isOrderTimeNull, bytes, Long.BYTES); - longToBytes(longValues.get(groupId), bytes, Long.BYTES + 1); - return bytes; - case FLOAT: - length += Float.BYTES; - bytes = new byte[length]; - longToBytes(minTimes.get(groupId), bytes, 0); - boolToBytes(isOrderTimeNull, bytes, Long.BYTES); - floatToBytes(floatValues.get(groupId), bytes, Long.BYTES + 1); - return bytes; - case DOUBLE: - length += Double.BYTES; - bytes = new byte[length]; - longToBytes(minTimes.get(groupId), bytes, 0); - boolToBytes(isOrderTimeNull, bytes, Long.BYTES); - doubleToBytes(doubleValues.get(groupId), bytes, Long.BYTES + 1); - return bytes; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - byte[] values = binaryValues.get(groupId).getValues(); - length += Integer.BYTES + values.length; - bytes = new byte[length]; - longToBytes(minTimes.get(groupId), bytes, 0); - boolToBytes(isOrderTimeNull, bytes, Long.BYTES); - BytesUtils.intToBytes(values.length, bytes, Long.BYTES + 1); - System.arraycopy(values, 0, bytes, length - values.length, values.length); - return bytes; - case BOOLEAN: - length++; - bytes = new byte[length]; - longToBytes(minTimes.get(groupId), bytes, 0); - boolToBytes(isOrderTimeNull, bytes, Long.BYTES); - boolToBytes(booleanValues.get(groupId), bytes, Long.BYTES + 1); - return bytes; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_B2D27BB9, - seriesDataType)); - } + byte[] bytes = new byte[length]; + longToBytes(minTimes.get(groupId), bytes, 0); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + valueSerializer.serialize(this, groupId, bytes, Long.BYTES + 1); + return bytes; } private boolean checkAndUpdateFirstTime(int groupId, long curTime) { @@ -670,4 +388,124 @@ protected void updateBooleanNullTimeValue(int groupId, boolean value) { booleanValues.set(groupId, value); } } + + @Override + public RuntimeException unsupportedException() { + return new UnSupportedDataTypeException( + String.format(CalcMessages.UNSUPPORTED_DATA_TYPE_IN_FIRST_AGGREGATION, seriesDataType)); + } + + @Override + public void initializeIntValues() { + intValues = new IntBigArray(); + } + + @Override + public void initializeLongValues() { + longValues = new LongBigArray(); + } + + @Override + public void initializeFloatValues() { + floatValues = new FloatBigArray(); + } + + @Override + public void initializeDoubleValues() { + doubleValues = new DoubleBigArray(); + } + + @Override + public void initializeBinaryValues() { + binaryValues = new BinaryBigArray(); + } + + @Override + public void initializeBooleanValues() { + booleanValues = new BooleanBigArray(); + } + + @Override + public IntBigArray getIntValues() { + return intValues; + } + + @Override + public LongBigArray getLongValues() { + return longValues; + } + + @Override + public FloatBigArray getFloatValues() { + return floatValues; + } + + @Override + public DoubleBigArray getDoubleValues() { + return doubleValues; + } + + @Override + public BinaryBigArray getBinaryValues() { + return binaryValues; + } + + @Override + public BooleanBigArray getBooleanValues() { + return booleanValues; + } + + @Override + public void updateInt(int groupId, int value, boolean timeNull, long time) { + if (timeNull) { + updateIntNullTimeValue(groupId, value); + } else { + updateIntValue(groupId, value, time); + } + } + + @Override + public void updateLong(int groupId, long value, boolean timeNull, long time) { + if (timeNull) { + updateLongNullTimeValue(groupId, value); + } else { + updateLongValue(groupId, value, time); + } + } + + @Override + public void updateFloat(int groupId, float value, boolean timeNull, long time) { + if (timeNull) { + updateFloatNullTimeValue(groupId, value); + } else { + updateFloatValue(groupId, value, time); + } + } + + @Override + public void updateDouble(int groupId, double value, boolean timeNull, long time) { + if (timeNull) { + updateDoubleNullTimeValue(groupId, value); + } else { + updateDoubleValue(groupId, value, time); + } + } + + @Override + public void updateBinary(int groupId, Binary value, boolean timeNull, long time) { + if (timeNull) { + updateBinaryNullTimeValue(groupId, value); + } else { + updateBinaryValue(groupId, value, time); + } + } + + @Override + public void updateBoolean(int groupId, boolean value, boolean timeNull, long time) { + if (timeNull) { + updateBooleanNullTimeValue(groupId, value); + } else { + updateBooleanValue(groupId, value, time); + } + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java index 378f188cf734..0bcbc90d4fe6 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java @@ -27,6 +27,7 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -34,6 +35,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -41,18 +43,22 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.tsfile.utils.BytesUtils.boolToBytes; -import static org.apache.tsfile.utils.BytesUtils.doubleToBytes; -import static org.apache.tsfile.utils.BytesUtils.floatToBytes; -import static org.apache.tsfile.utils.BytesUtils.intToBytes; import static org.apache.tsfile.utils.BytesUtils.longToBytes; -public class GroupedFirstByAccumulator implements GroupedAccumulator { +public class GroupedFirstByAccumulator + implements GroupedAccumulator, + TypeServices.GroupedValueAccessor, + TypeServices.GroupedByValueConsumer { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedFirstByAccumulator.class); private final TSDataType xDataType; private final TSDataType yDataType; + private final TypeServices.GroupedValueService valueService; + private final TypeServices.GroupedValueSerializer valueSerializer; + private final TypeServices.GroupedByInputReader inputReader; + private final TypeServices.GroupedByValueDeserializer valueDeserializer; private final LongBigArray yFirstTimes = new LongBigArray(Long.MAX_VALUE); @@ -71,75 +77,18 @@ public class GroupedFirstByAccumulator implements GroupedAccumulator { public GroupedFirstByAccumulator(TSDataType xDataType, TSDataType yDataType) { this.xDataType = xDataType; this.yDataType = yDataType; - - switch (xDataType) { - case INT32: - case DATE: - xIntValues = new IntBigArray(); - break; - case INT64: - case TIMESTAMP: - xLongValues = new LongBigArray(); - break; - case FLOAT: - xFloatValues = new FloatBigArray(); - break; - case DOUBLE: - xDoubleValues = new DoubleBigArray(); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - xBinaryValues = new BinaryBigArray(); - break; - case BOOLEAN: - xBooleanValues = new BooleanBigArray(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - xDataType)); - } + Type type = Type.fromTsDataType(xDataType); + this.valueService = TypeServices.GROUPED_VALUE_SERVICE.call(type); + this.valueSerializer = TypeServices.GROUPED_VALUE_SERIALIZER_SERVICE.call(type); + this.inputReader = TypeServices.GROUPED_BY_INPUT_READER_SERVICE.call(type); + this.valueDeserializer = TypeServices.GROUPED_BY_VALUE_DESERIALIZER_SERVICE.call(type); + valueService.initialize(this); } @Override public long getEstimatedSize() { - long valuesSize = 0; - switch (xDataType) { - case INT32: - case DATE: - valuesSize += xIntValues.sizeOf(); - break; - case INT64: - case TIMESTAMP: - valuesSize += xLongValues.sizeOf(); - break; - case FLOAT: - valuesSize += xFloatValues.sizeOf(); - break; - case DOUBLE: - valuesSize += xDoubleValues.sizeOf(); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - valuesSize += xBinaryValues.sizeOf(); - break; - case BOOLEAN: - valuesSize += xBooleanValues.sizeOf(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - xDataType)); - } - return INSTANCE_SIZE - + valuesSize + + valueService.sizeOf(this) + yFirstTimes.sizeOf() + inits.sizeOf() + initNullTimeValues.sizeOf() @@ -152,36 +101,7 @@ public void setGroupCount(long groupCount) { inits.ensureCapacity(groupCount); initNullTimeValues.ensureCapacity(groupCount); xNulls.ensureCapacity(groupCount); - switch (xDataType) { - case INT32: - case DATE: - xIntValues.ensureCapacity(groupCount); - return; - case INT64: - case TIMESTAMP: - xLongValues.ensureCapacity(groupCount); - return; - case FLOAT: - xFloatValues.ensureCapacity(groupCount); - return; - case DOUBLE: - xDoubleValues.ensureCapacity(groupCount); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - xBinaryValues.ensureCapacity(groupCount); - return; - case BOOLEAN: - xBooleanValues.ensureCapacity(groupCount); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - xDataType)); - } + valueService.ensureCapacity(this, groupCount); } @Override @@ -193,36 +113,7 @@ public void reset() { inits.reset(); initNullTimeValues.reset(); xNulls.reset(); - switch (xDataType) { - case INT32: - case DATE: - xIntValues.reset(); - break; - case INT64: - case TIMESTAMP: - xLongValues.reset(); - break; - case FLOAT: - xFloatValues.reset(); - break; - case DOUBLE: - xDoubleValues.reset(); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - xBinaryValues.reset(); - break; - case BOOLEAN: - xBooleanValues.reset(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - xDataType)); - } + valueService.reset(this); } @Override @@ -233,36 +124,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { .EXCEPTION_LENGTH_OF_INPUT_COLUMN_LEFT_BRACKET_RIGHT_BRACKET_FOR_LASTBY_SLASH_FIRSTBY_SHOUL_9E836ABA); // arguments[0] is x column, arguments[1] is y column, arguments[2] is time column - switch (xDataType) { - case INT32: - case DATE: - addIntInput(groupIds, arguments[0], arguments[1], arguments[2], mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(groupIds, arguments[0], arguments[1], arguments[2], mask); - return; - case FLOAT: - addFloatInput(groupIds, arguments[0], arguments[1], arguments[2], mask); - return; - case DOUBLE: - addDoubleInput(groupIds, arguments[0], arguments[1], arguments[2], mask); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(groupIds, arguments[0], arguments[1], arguments[2], mask); - return; - case BOOLEAN: - addBooleanInput(groupIds, arguments[0], arguments[1], arguments[2], mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - xDataType)); - } + inputReader.addInput(groupIds, arguments, mask, this); } @Override @@ -288,75 +150,8 @@ public void addIntermediate(int[] groupIds, Column argument) { offset += 1; int groupId = groupIds[i]; - switch (xDataType) { - case INT32: - case DATE: - int intVal = isXValueNull ? 0 : BytesUtils.bytesToInt(bytes, offset); - if (!isOrderTimeNull) { - updateIntFirstValue(groupId, isXValueNull, intVal, curTime); - } else { - updateIntNullTimeValue(groupId, isXValueNull, intVal); - } - break; - case INT64: - case TIMESTAMP: - long longVal = - isXValueNull ? 0 : BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - if (!isOrderTimeNull) { - updateLongFirstValue(groupId, isXValueNull, longVal, curTime); - } else { - updateLongNullTimeValue(groupId, isXValueNull, longVal); - } - break; - case FLOAT: - float floatVal = isXValueNull ? 0 : BytesUtils.bytesToFloat(bytes, offset); - if (!isOrderTimeNull) { - updateFloatFirstValue(groupId, isXValueNull, floatVal, curTime); - } else { - updateFloatNullTimeValue(groupId, isXValueNull, floatVal); - } - break; - case DOUBLE: - double doubleVal = isXValueNull ? 0 : BytesUtils.bytesToDouble(bytes, offset); - if (!isOrderTimeNull) { - updateDoubleFirstValue(groupId, isXValueNull, doubleVal, curTime); - } else { - updateDoubleNullTimeValue(groupId, isXValueNull, doubleVal); - } - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - Binary binaryVal = null; - if (!isXValueNull) { - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - } - if (!isOrderTimeNull) { - updateBinaryFirstValue(groupId, isXValueNull, binaryVal, curTime); - } else { - updateBinaryNullTimeValue(groupId, isXValueNull, binaryVal); - } - break; - case BOOLEAN: - boolean boolVal = false; - if (!isXValueNull) { - boolVal = BytesUtils.bytesToBool(bytes, offset); - } - if (!isOrderTimeNull) { - updateBooleanFirstValue(groupId, isXValueNull, boolVal, curTime); - } else { - updateBooleanNullTimeValue(groupId, isXValueNull, boolVal); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - xDataType)); - } + valueDeserializer.deserialize( + bytes, offset, groupId, isXValueNull, curTime, isOrderTimeNull, this); } } @@ -377,7 +172,7 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { /** Serializes group state: Time (8B) | xNull (1B) | OrderTimeNull (1B) | [Value] */ private byte[] serializeTimeWithValue(int groupId) { boolean xNull = xNulls.get(groupId); - int length = Long.BYTES + 2 + (xNull ? 0 : calculateValueLength(groupId)); + int length = Long.BYTES + 2 + (xNull ? 0 : valueSerializer.calcTypeSize(this, groupId)); byte[] bytes = new byte[length]; boolean isOrderTimeNull = !inits.get(groupId); @@ -387,69 +182,11 @@ private byte[] serializeTimeWithValue(int groupId) { if (!xNull) { int valueOffset = Long.BYTES + 2; - switch (xDataType) { - case INT32: - case DATE: - intToBytes(xIntValues.get(groupId), bytes, valueOffset); - return bytes; - case INT64: - case TIMESTAMP: - longToBytes(xLongValues.get(groupId), bytes, valueOffset); - return bytes; - case FLOAT: - floatToBytes(xFloatValues.get(groupId), bytes, valueOffset); - return bytes; - case DOUBLE: - doubleToBytes(xDoubleValues.get(groupId), bytes, valueOffset); - return bytes; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - byte[] values = xBinaryValues.get(groupId).getValues(); - intToBytes(values.length, bytes, valueOffset); - System.arraycopy(values, 0, bytes, length - values.length, values.length); - return bytes; - case BOOLEAN: - boolToBytes(xBooleanValues.get(groupId), bytes, valueOffset); - return bytes; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - xDataType)); - } + valueSerializer.serialize(this, groupId, bytes, valueOffset); } return bytes; } - private int calculateValueLength(int groupId) { - switch (xDataType) { - case INT32: - case DATE: - return Integer.BYTES; - case INT64: - case TIMESTAMP: - return Long.BYTES; - case FLOAT: - return Float.BYTES; - case DOUBLE: - return Double.BYTES; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - return Integer.BYTES + xBinaryValues.get(groupId).getValues().length; - case BOOLEAN: - return 1; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - xDataType)); - } - } - @Override public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { if (xNulls.get(groupId)) { @@ -457,36 +194,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { return; } - switch (xDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(xIntValues.get(groupId)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(xLongValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(xFloatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(xDoubleValues.get(groupId)); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - columnBuilder.writeBinary(xBinaryValues.get(groupId)); - break; - case BOOLEAN: - columnBuilder.writeBoolean(xBooleanValues.get(groupId)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_FIRST_AGGREGATION_ARG_8D31C6CA, - xDataType)); - } + valueService.write(this, groupId, columnBuilder); } private boolean checkAndUpdateFirstTime(int groupId, boolean isXValueNull, long curTime) { @@ -752,4 +460,125 @@ protected void updateBooleanNullTimeValue(int groupId, boolean isXValueNull, boo xBooleanValues.set(groupId, xValue); } } + + @Override + public RuntimeException unsupportedException() { + return new UnSupportedDataTypeException( + String.format(CalcMessages.UNSUPPORTED_DATA_TYPE_IN_FIRST_BY_AGGREGATION, xDataType)); + } + + @Override + public void initializeIntValues() { + xIntValues = new IntBigArray(); + } + + @Override + public void initializeLongValues() { + xLongValues = new LongBigArray(); + } + + @Override + public void initializeFloatValues() { + xFloatValues = new FloatBigArray(); + } + + @Override + public void initializeDoubleValues() { + xDoubleValues = new DoubleBigArray(); + } + + @Override + public void initializeBinaryValues() { + xBinaryValues = new BinaryBigArray(); + } + + @Override + public void initializeBooleanValues() { + xBooleanValues = new BooleanBigArray(); + } + + @Override + public IntBigArray getIntValues() { + return xIntValues; + } + + @Override + public LongBigArray getLongValues() { + return xLongValues; + } + + @Override + public FloatBigArray getFloatValues() { + return xFloatValues; + } + + @Override + public DoubleBigArray getDoubleValues() { + return xDoubleValues; + } + + @Override + public BinaryBigArray getBinaryValues() { + return xBinaryValues; + } + + @Override + public BooleanBigArray getBooleanValues() { + return xBooleanValues; + } + + @Override + public void updateInt(int groupId, boolean xNull, int value, boolean timeNull, long time) { + if (timeNull) { + updateIntNullTimeValue(groupId, xNull, value); + } else { + updateIntFirstValue(groupId, xNull, value, time); + } + } + + @Override + public void updateLong(int groupId, boolean xNull, long value, boolean timeNull, long time) { + if (timeNull) { + updateLongNullTimeValue(groupId, xNull, value); + } else { + updateLongFirstValue(groupId, xNull, value, time); + } + } + + @Override + public void updateFloat(int groupId, boolean xNull, float value, boolean timeNull, long time) { + if (timeNull) { + updateFloatNullTimeValue(groupId, xNull, value); + } else { + updateFloatFirstValue(groupId, xNull, value, time); + } + } + + @Override + public void updateDouble(int groupId, boolean xNull, double value, boolean timeNull, long time) { + if (timeNull) { + updateDoubleNullTimeValue(groupId, xNull, value); + } else { + updateDoubleFirstValue(groupId, xNull, value, time); + } + } + + @Override + public void updateBinary(int groupId, boolean xNull, Binary value, boolean timeNull, long time) { + if (timeNull) { + updateBinaryNullTimeValue(groupId, xNull, value); + } else { + updateBinaryFirstValue(groupId, xNull, value, time); + } + } + + @Override + public void updateBoolean( + int groupId, boolean xNull, boolean value, boolean timeNull, long time) { + if (timeNull) { + updateBooleanNullTimeValue(groupId, xNull, value); + } else { + updateBooleanFirstValue(groupId, xNull, value, time); + } + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java index abdd939b5825..1e23cc935b6d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java @@ -27,6 +27,7 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -34,6 +35,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -41,15 +43,19 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.tsfile.utils.BytesUtils.boolToBytes; -import static org.apache.tsfile.utils.BytesUtils.doubleToBytes; -import static org.apache.tsfile.utils.BytesUtils.floatToBytes; -import static org.apache.tsfile.utils.BytesUtils.intToBytes; import static org.apache.tsfile.utils.BytesUtils.longToBytes; -public class GroupedLastAccumulator implements GroupedAccumulator { +public class GroupedLastAccumulator + implements GroupedAccumulator, + TypeServices.GroupedValueAccessor, + TypeServices.GroupedTimeValueConsumer { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedLastAccumulator.class); private final TSDataType seriesDataType; + private final TypeServices.GroupedValueService valueService; + private final TypeServices.GroupedValueSerializer valueSerializer; + private final TypeServices.GroupedTimeInputReader inputReader; + private final TypeServices.GroupedTimeValueDeserializer valueDeserializer; private final LongBigArray maxTimes = new LongBigArray(Long.MIN_VALUE); private final BooleanBigArray inits = new BooleanBigArray(); private final BooleanBigArray initNullTimeValues = new BooleanBigArray(); @@ -63,77 +69,21 @@ public class GroupedLastAccumulator implements GroupedAccumulator { public GroupedLastAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - switch (seriesDataType) { - case INT32: - case DATE: - intValues = new IntBigArray(); - return; - case INT64: - case TIMESTAMP: - longValues = new LongBigArray(); - return; - case FLOAT: - floatValues = new FloatBigArray(); - return; - case DOUBLE: - doubleValues = new DoubleBigArray(); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - binaryValues = new BinaryBigArray(); - return; - case BOOLEAN: - booleanValues = new BooleanBigArray(); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_2FCF5ADA, - seriesDataType)); - } + Type type = Type.fromTsDataType(seriesDataType); + this.valueService = TypeServices.GROUPED_VALUE_SERVICE.call(type); + this.valueSerializer = TypeServices.GROUPED_VALUE_SERIALIZER_SERVICE.call(type); + this.inputReader = TypeServices.GROUPED_TIME_INPUT_READER_SERVICE.call(type); + this.valueDeserializer = TypeServices.GROUPED_TIME_VALUE_DESERIALIZER_SERVICE.call(type); + valueService.initialize(this); } @Override public long getEstimatedSize() { - long valuesSize = 0; - switch (seriesDataType) { - case INT32: - case DATE: - valuesSize += intValues.sizeOf(); - break; - case INT64: - case TIMESTAMP: - valuesSize += longValues.sizeOf(); - break; - case FLOAT: - valuesSize += floatValues.sizeOf(); - break; - case DOUBLE: - valuesSize += doubleValues.sizeOf(); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - valuesSize += binaryValues.sizeOf(); - break; - case BOOLEAN: - valuesSize += booleanValues.sizeOf(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_2FCF5ADA, - seriesDataType)); - } - return INSTANCE_SIZE + inits.sizeOf() + maxTimes.sizeOf() + initNullTimeValues.sizeOf() - + valuesSize; + + valueService.sizeOf(this); } @Override @@ -141,71 +91,13 @@ public void setGroupCount(long groupCount) { maxTimes.ensureCapacity(groupCount); inits.ensureCapacity(groupCount); initNullTimeValues.ensureCapacity(groupCount); - switch (seriesDataType) { - case INT32: - case DATE: - intValues.ensureCapacity(groupCount); - return; - case INT64: - case TIMESTAMP: - longValues.ensureCapacity(groupCount); - return; - case FLOAT: - floatValues.ensureCapacity(groupCount); - return; - case DOUBLE: - doubleValues.ensureCapacity(groupCount); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - binaryValues.ensureCapacity(groupCount); - return; - case BOOLEAN: - booleanValues.ensureCapacity(groupCount); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_2FCF5ADA, - seriesDataType)); - } + valueService.ensureCapacity(this, groupCount); } @Override public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { // arguments[0] is value column, arguments[1] is time column - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(groupIds, arguments[0], arguments[1], mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(groupIds, arguments[0], arguments[1], mask); - return; - case FLOAT: - addFloatInput(groupIds, arguments[0], arguments[1], mask); - return; - case DOUBLE: - addDoubleInput(groupIds, arguments[0], arguments[1], mask); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(groupIds, arguments[0], arguments[1], mask); - return; - case BOOLEAN: - addBooleanInput(groupIds, arguments[0], arguments[1], mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_2FCF5ADA, - seriesDataType)); - } + inputReader.addInput(groupIds, arguments, mask, this); } @Override @@ -229,68 +121,7 @@ public void addIntermediate(int[] groupIds, Column argument) { offset += 1; int groupId = groupIds[i]; - switch (seriesDataType) { - case INT32: - case DATE: - int intVal = BytesUtils.bytesToInt(bytes, offset); - if (!isOrderTimeNull) { - updateIntValue(groupId, intVal, time); - } else { - updateIntNullTimeValue(groupId, intVal); - } - break; - case INT64: - case TIMESTAMP: - long longVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - if (!isOrderTimeNull) { - updateLongValue(groupId, longVal, time); - } else { - updateLongNullTimeValue(groupId, longVal); - } - break; - case FLOAT: - float floatVal = BytesUtils.bytesToFloat(bytes, offset); - if (!isOrderTimeNull) { - updateFloatValue(groupId, floatVal, time); - } else { - updateFloatNullTimeValue(groupId, floatVal); - } - break; - case DOUBLE: - double doubleVal = BytesUtils.bytesToDouble(bytes, offset); - if (!isOrderTimeNull) { - updateDoubleValue(groupId, doubleVal, time); - } else { - updateDoubleNullTimeValue(groupId, doubleVal); - } - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - if (!isOrderTimeNull) { - updateBinaryValue(groupId, binaryVal, time); - } else { - updateBinaryNullTimeValue(groupId, binaryVal); - } - break; - case BOOLEAN: - boolean boolVal = BytesUtils.bytesToBool(bytes, offset); - if (!isOrderTimeNull) { - updateBooleanValue(groupId, boolVal, time); - } else { - updateBooleanNullTimeValue(groupId, boolVal); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_2FCF5ADA, - seriesDataType)); - } + valueDeserializer.deserialize(bytes, offset, groupId, time, isOrderTimeNull, this); } } @@ -314,36 +145,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(intValues.get(groupId)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(longValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(floatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(doubleValues.get(groupId)); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - columnBuilder.writeBinary(binaryValues.get(groupId)); - break; - case BOOLEAN: - columnBuilder.writeBoolean(booleanValues.get(groupId)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_2FCF5ADA, - seriesDataType)); - } + valueService.write(this, groupId, columnBuilder); } @Override @@ -354,100 +156,17 @@ public void reset() { maxTimes.reset(); inits.reset(); initNullTimeValues.reset(); - switch (seriesDataType) { - case INT32: - case DATE: - intValues.reset(); - return; - case INT64: - case TIMESTAMP: - longValues.reset(); - return; - case FLOAT: - floatValues.reset(); - return; - case DOUBLE: - doubleValues.reset(); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - binaryValues.reset(); - return; - case BOOLEAN: - booleanValues.reset(); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_2FCF5ADA, - seriesDataType)); - } + valueService.reset(this); } private byte[] serializeTimeWithValue(int groupId) { - byte[] bytes; - int length = Long.BYTES; + int length = Long.BYTES + 1 + valueSerializer.calcTypeSize(this, groupId); boolean isOrderTimeNull = !inits.get(groupId); - length += 1; - - switch (seriesDataType) { - case INT32: - case DATE: - length += Integer.BYTES; - bytes = new byte[length]; - longToBytes(maxTimes.get(groupId), bytes, 0); - boolToBytes(isOrderTimeNull, bytes, Long.BYTES); - intToBytes(intValues.get(groupId), bytes, Long.BYTES + 1); - return bytes; - case INT64: - case TIMESTAMP: - length += Long.BYTES; - bytes = new byte[length]; - longToBytes(maxTimes.get(groupId), bytes, 0); - boolToBytes(isOrderTimeNull, bytes, Long.BYTES); - longToBytes(longValues.get(groupId), bytes, Long.BYTES + 1); - return bytes; - case FLOAT: - length += Float.BYTES; - bytes = new byte[length]; - longToBytes(maxTimes.get(groupId), bytes, 0); - boolToBytes(isOrderTimeNull, bytes, Long.BYTES); - floatToBytes(floatValues.get(groupId), bytes, Long.BYTES + 1); - return bytes; - case DOUBLE: - length += Double.BYTES; - bytes = new byte[length]; - longToBytes(maxTimes.get(groupId), bytes, 0); - boolToBytes(isOrderTimeNull, bytes, Long.BYTES); - doubleToBytes(doubleValues.get(groupId), bytes, Long.BYTES + 1); - return bytes; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - byte[] values = binaryValues.get(groupId).getValues(); - length += Integer.BYTES + values.length; - bytes = new byte[length]; - longToBytes(maxTimes.get(groupId), bytes, 0); - boolToBytes(isOrderTimeNull, bytes, Long.BYTES); - BytesUtils.intToBytes(values.length, bytes, Long.BYTES + 1); - System.arraycopy(values, 0, bytes, length - values.length, values.length); - return bytes; - case BOOLEAN: - length++; - bytes = new byte[length]; - longToBytes(maxTimes.get(groupId), bytes, 0); - boolToBytes(isOrderTimeNull, bytes, Long.BYTES); - boolToBytes(booleanValues.get(groupId), bytes, Long.BYTES + 1); - return bytes; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_2FCF5ADA, - seriesDataType)); - } + byte[] bytes = new byte[length]; + longToBytes(maxTimes.get(groupId), bytes, 0); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + valueSerializer.serialize(this, groupId, bytes, Long.BYTES + 1); + return bytes; } private boolean checkAndUpdateLastTime(int groupId, long curTime) { @@ -670,4 +389,124 @@ protected void updateBooleanNullTimeValue(int groupId, boolean value) { booleanValues.set(groupId, value); } } + + @Override + public RuntimeException unsupportedException() { + return new UnSupportedDataTypeException( + String.format(CalcMessages.UNSUPPORTED_DATA_TYPE_IN_LAST_AGGREGATION, seriesDataType)); + } + + @Override + public void initializeIntValues() { + intValues = new IntBigArray(); + } + + @Override + public void initializeLongValues() { + longValues = new LongBigArray(); + } + + @Override + public void initializeFloatValues() { + floatValues = new FloatBigArray(); + } + + @Override + public void initializeDoubleValues() { + doubleValues = new DoubleBigArray(); + } + + @Override + public void initializeBinaryValues() { + binaryValues = new BinaryBigArray(); + } + + @Override + public void initializeBooleanValues() { + booleanValues = new BooleanBigArray(); + } + + @Override + public IntBigArray getIntValues() { + return intValues; + } + + @Override + public LongBigArray getLongValues() { + return longValues; + } + + @Override + public FloatBigArray getFloatValues() { + return floatValues; + } + + @Override + public DoubleBigArray getDoubleValues() { + return doubleValues; + } + + @Override + public BinaryBigArray getBinaryValues() { + return binaryValues; + } + + @Override + public BooleanBigArray getBooleanValues() { + return booleanValues; + } + + @Override + public void updateInt(int groupId, int value, boolean timeNull, long time) { + if (timeNull) { + updateIntNullTimeValue(groupId, value); + } else { + updateIntValue(groupId, value, time); + } + } + + @Override + public void updateLong(int groupId, long value, boolean timeNull, long time) { + if (timeNull) { + updateLongNullTimeValue(groupId, value); + } else { + updateLongValue(groupId, value, time); + } + } + + @Override + public void updateFloat(int groupId, float value, boolean timeNull, long time) { + if (timeNull) { + updateFloatNullTimeValue(groupId, value); + } else { + updateFloatValue(groupId, value, time); + } + } + + @Override + public void updateDouble(int groupId, double value, boolean timeNull, long time) { + if (timeNull) { + updateDoubleNullTimeValue(groupId, value); + } else { + updateDoubleValue(groupId, value, time); + } + } + + @Override + public void updateBinary(int groupId, Binary value, boolean timeNull, long time) { + if (timeNull) { + updateBinaryNullTimeValue(groupId, value); + } else { + updateBinaryValue(groupId, value, time); + } + } + + @Override + public void updateBoolean(int groupId, boolean value, boolean timeNull, long time) { + if (timeNull) { + updateBooleanNullTimeValue(groupId, value); + } else { + updateBooleanValue(groupId, value, time); + } + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java index fd29bbdb376b..5f709cbc7f0f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java @@ -27,6 +27,7 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -34,6 +35,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -41,18 +43,22 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.tsfile.utils.BytesUtils.boolToBytes; -import static org.apache.tsfile.utils.BytesUtils.doubleToBytes; -import static org.apache.tsfile.utils.BytesUtils.floatToBytes; -import static org.apache.tsfile.utils.BytesUtils.intToBytes; import static org.apache.tsfile.utils.BytesUtils.longToBytes; -public class GroupedLastByAccumulator implements GroupedAccumulator { +public class GroupedLastByAccumulator + implements GroupedAccumulator, + TypeServices.GroupedValueAccessor, + TypeServices.GroupedByValueConsumer { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedLastByAccumulator.class); private final TSDataType xDataType; private final TSDataType yDataType; + private final TypeServices.GroupedValueService valueService; + private final TypeServices.GroupedValueSerializer valueSerializer; + private final TypeServices.GroupedByInputReader inputReader; + private final TypeServices.GroupedByValueDeserializer valueDeserializer; private final LongBigArray yLastTimes = new LongBigArray(Long.MIN_VALUE); @@ -71,75 +77,18 @@ public class GroupedLastByAccumulator implements GroupedAccumulator { public GroupedLastByAccumulator(TSDataType xDataType, TSDataType yDataType) { this.xDataType = xDataType; this.yDataType = yDataType; - - switch (xDataType) { - case INT32: - case DATE: - xIntValues = new IntBigArray(); - break; - case INT64: - case TIMESTAMP: - xLongValues = new LongBigArray(); - break; - case FLOAT: - xFloatValues = new FloatBigArray(); - break; - case DOUBLE: - xDoubleValues = new DoubleBigArray(); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - xBinaryValues = new BinaryBigArray(); - break; - case BOOLEAN: - xBooleanValues = new BooleanBigArray(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - xDataType)); - } + Type type = Type.fromTsDataType(xDataType); + this.valueService = TypeServices.GROUPED_VALUE_SERVICE.call(type); + this.valueSerializer = TypeServices.GROUPED_VALUE_SERIALIZER_SERVICE.call(type); + this.inputReader = TypeServices.GROUPED_BY_INPUT_READER_SERVICE.call(type); + this.valueDeserializer = TypeServices.GROUPED_BY_VALUE_DESERIALIZER_SERVICE.call(type); + valueService.initialize(this); } @Override public long getEstimatedSize() { - long valuesSize = 0; - switch (xDataType) { - case INT32: - case DATE: - valuesSize += xIntValues.sizeOf(); - break; - case INT64: - case TIMESTAMP: - valuesSize += xLongValues.sizeOf(); - break; - case FLOAT: - valuesSize += xFloatValues.sizeOf(); - break; - case DOUBLE: - valuesSize += xDoubleValues.sizeOf(); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - valuesSize += xBinaryValues.sizeOf(); - break; - case BOOLEAN: - valuesSize += xBooleanValues.sizeOf(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - xDataType)); - } - return INSTANCE_SIZE - + valuesSize + + valueService.sizeOf(this) + yLastTimes.sizeOf() + inits.sizeOf() + initNullTimeValues.sizeOf() @@ -152,36 +101,7 @@ public void setGroupCount(long groupCount) { inits.ensureCapacity(groupCount); initNullTimeValues.ensureCapacity(groupCount); xNulls.ensureCapacity(groupCount); - switch (xDataType) { - case INT32: - case DATE: - xIntValues.ensureCapacity(groupCount); - return; - case INT64: - case TIMESTAMP: - xLongValues.ensureCapacity(groupCount); - return; - case FLOAT: - xFloatValues.ensureCapacity(groupCount); - return; - case DOUBLE: - xDoubleValues.ensureCapacity(groupCount); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - xBinaryValues.ensureCapacity(groupCount); - return; - case BOOLEAN: - xBooleanValues.ensureCapacity(groupCount); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - xDataType)); - } + valueService.ensureCapacity(this, groupCount); } @Override @@ -193,36 +113,7 @@ public void reset() { inits.reset(); initNullTimeValues.reset(); xNulls.reset(); - switch (xDataType) { - case INT32: - case DATE: - xIntValues.reset(); - break; - case INT64: - case TIMESTAMP: - xLongValues.reset(); - break; - case FLOAT: - xFloatValues.reset(); - break; - case DOUBLE: - xDoubleValues.reset(); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - xBinaryValues.reset(); - break; - case BOOLEAN: - xBooleanValues.reset(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - xDataType)); - } + valueService.reset(this); } @Override @@ -233,36 +124,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { .EXCEPTION_LENGTH_OF_INPUT_COLUMN_LEFT_BRACKET_RIGHT_BRACKET_FOR_LAST_BY_SHOULD_BE_3_D759D45F); // arguments[0] is x column, arguments[1] is y column, arguments[2] is time column - switch (xDataType) { - case INT32: - case DATE: - addIntInput(groupIds, arguments[0], arguments[1], arguments[2], mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(groupIds, arguments[0], arguments[1], arguments[2], mask); - return; - case FLOAT: - addFloatInput(groupIds, arguments[0], arguments[1], arguments[2], mask); - return; - case DOUBLE: - addDoubleInput(groupIds, arguments[0], arguments[1], arguments[2], mask); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(groupIds, arguments[0], arguments[1], arguments[2], mask); - return; - case BOOLEAN: - addBooleanInput(groupIds, arguments[0], arguments[1], arguments[2], mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - xDataType)); - } + inputReader.addInput(groupIds, arguments, mask, this); } @Override @@ -288,75 +150,8 @@ public void addIntermediate(int[] groupIds, Column argument) { offset += 1; int groupId = groupIds[i]; - switch (xDataType) { - case INT32: - case DATE: - int intVal = isXValueNull ? 0 : BytesUtils.bytesToInt(bytes, offset); - if (!isOrderTimeNull) { - updateIntLastValue(groupId, isXValueNull, intVal, curTime); - } else { - updateIntNullTimeValue(groupId, isXValueNull, intVal); - } - break; - case INT64: - case TIMESTAMP: - long longVal = - isXValueNull ? 0 : BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - if (!isOrderTimeNull) { - updateLongLastValue(groupId, isXValueNull, longVal, curTime); - } else { - updateLongNullTimeValue(groupId, isXValueNull, longVal); - } - break; - case FLOAT: - float floatVal = isXValueNull ? 0 : BytesUtils.bytesToFloat(bytes, offset); - if (!isOrderTimeNull) { - updateFloatLastValue(groupId, isXValueNull, floatVal, curTime); - } else { - updateFloatNullTimeValue(groupId, isXValueNull, floatVal); - } - break; - case DOUBLE: - double doubleVal = isXValueNull ? 0 : BytesUtils.bytesToDouble(bytes, offset); - if (!isOrderTimeNull) { - updateDoubleLastValue(groupId, isXValueNull, doubleVal, curTime); - } else { - updateDoubleNullTimeValue(groupId, isXValueNull, doubleVal); - } - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - Binary binaryVal = null; - if (!isXValueNull) { - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - } - if (!isOrderTimeNull) { - updateBinaryLastValue(groupId, isXValueNull, binaryVal, curTime); - } else { - updateBinaryNullTimeValue(groupId, isXValueNull, binaryVal); - } - break; - case BOOLEAN: - boolean boolVal = false; - if (!isXValueNull) { - boolVal = BytesUtils.bytesToBool(bytes, offset); - } - if (!isOrderTimeNull) { - updateBooleanLastValue(groupId, isXValueNull, boolVal, curTime); - } else { - updateBooleanNullTimeValue(groupId, isXValueNull, boolVal); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - xDataType)); - } + valueDeserializer.deserialize( + bytes, offset, groupId, isXValueNull, curTime, isOrderTimeNull, this); } } @@ -376,7 +171,7 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { private byte[] serializeTimeWithValue(int groupId) { boolean xNull = xNulls.get(groupId); - int length = Long.BYTES + 2 + (xNull ? 0 : calculateValueLength(groupId)); + int length = Long.BYTES + 2 + (xNull ? 0 : valueSerializer.calcTypeSize(this, groupId)); byte[] bytes = new byte[length]; boolean isOrderTimeNull = !inits.get(groupId); @@ -386,69 +181,11 @@ private byte[] serializeTimeWithValue(int groupId) { if (!xNull) { int valueOffset = Long.BYTES + 2; - switch (xDataType) { - case INT32: - case DATE: - intToBytes(xIntValues.get(groupId), bytes, valueOffset); - return bytes; - case INT64: - case TIMESTAMP: - longToBytes(xLongValues.get(groupId), bytes, valueOffset); - return bytes; - case FLOAT: - floatToBytes(xFloatValues.get(groupId), bytes, valueOffset); - return bytes; - case DOUBLE: - doubleToBytes(xDoubleValues.get(groupId), bytes, valueOffset); - return bytes; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - byte[] values = xBinaryValues.get(groupId).getValues(); - intToBytes(values.length, bytes, valueOffset); - System.arraycopy(values, 0, bytes, length - values.length, values.length); - return bytes; - case BOOLEAN: - boolToBytes(xBooleanValues.get(groupId), bytes, valueOffset); - return bytes; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - xDataType)); - } + valueSerializer.serialize(this, groupId, bytes, valueOffset); } return bytes; } - private int calculateValueLength(int groupId) { - switch (xDataType) { - case INT32: - case DATE: - return Integer.BYTES; - case INT64: - case TIMESTAMP: - return Long.BYTES; - case FLOAT: - return Float.BYTES; - case DOUBLE: - return Double.BYTES; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - return Integer.BYTES + xBinaryValues.get(groupId).getValues().length; - case BOOLEAN: - return 1; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - xDataType)); - } - } - @Override public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { if (xNulls.get(groupId)) { @@ -456,36 +193,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { return; } - switch (xDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(xIntValues.get(groupId)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(xLongValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(xFloatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(xDoubleValues.get(groupId)); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - columnBuilder.writeBinary(xBinaryValues.get(groupId)); - break; - case BOOLEAN: - columnBuilder.writeBoolean(xBooleanValues.get(groupId)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_LAST_AGGREGATION_ARG_8C41FBCC, - xDataType)); - } + valueService.write(this, groupId, columnBuilder); } private boolean checkAndUpdateLastTime(int groupId, boolean isXValueNull, long curTime) { @@ -750,4 +458,125 @@ protected void updateBooleanNullTimeValue(int groupId, boolean isXValueNull, boo xBooleanValues.set(groupId, xValue); } } + + @Override + public RuntimeException unsupportedException() { + return new UnSupportedDataTypeException( + String.format(CalcMessages.UNSUPPORTED_DATA_TYPE_IN_LAST_BY_AGGREGATION, xDataType)); + } + + @Override + public void initializeIntValues() { + xIntValues = new IntBigArray(); + } + + @Override + public void initializeLongValues() { + xLongValues = new LongBigArray(); + } + + @Override + public void initializeFloatValues() { + xFloatValues = new FloatBigArray(); + } + + @Override + public void initializeDoubleValues() { + xDoubleValues = new DoubleBigArray(); + } + + @Override + public void initializeBinaryValues() { + xBinaryValues = new BinaryBigArray(); + } + + @Override + public void initializeBooleanValues() { + xBooleanValues = new BooleanBigArray(); + } + + @Override + public IntBigArray getIntValues() { + return xIntValues; + } + + @Override + public LongBigArray getLongValues() { + return xLongValues; + } + + @Override + public FloatBigArray getFloatValues() { + return xFloatValues; + } + + @Override + public DoubleBigArray getDoubleValues() { + return xDoubleValues; + } + + @Override + public BinaryBigArray getBinaryValues() { + return xBinaryValues; + } + + @Override + public BooleanBigArray getBooleanValues() { + return xBooleanValues; + } + + @Override + public void updateInt(int groupId, boolean xNull, int value, boolean timeNull, long time) { + if (timeNull) { + updateIntNullTimeValue(groupId, xNull, value); + } else { + updateIntLastValue(groupId, xNull, value, time); + } + } + + @Override + public void updateLong(int groupId, boolean xNull, long value, boolean timeNull, long time) { + if (timeNull) { + updateLongNullTimeValue(groupId, xNull, value); + } else { + updateLongLastValue(groupId, xNull, value, time); + } + } + + @Override + public void updateFloat(int groupId, boolean xNull, float value, boolean timeNull, long time) { + if (timeNull) { + updateFloatNullTimeValue(groupId, xNull, value); + } else { + updateFloatLastValue(groupId, xNull, value, time); + } + } + + @Override + public void updateDouble(int groupId, boolean xNull, double value, boolean timeNull, long time) { + if (timeNull) { + updateDoubleNullTimeValue(groupId, xNull, value); + } else { + updateDoubleLastValue(groupId, xNull, value, time); + } + } + + @Override + public void updateBinary(int groupId, boolean xNull, Binary value, boolean timeNull, long time) { + if (timeNull) { + updateBinaryNullTimeValue(groupId, xNull, value); + } else { + updateBinaryLastValue(groupId, xNull, value, time); + } + } + + @Override + public void updateBoolean( + int groupId, boolean xNull, boolean value, boolean timeNull, long time) { + if (timeNull) { + updateBooleanNullTimeValue(groupId, xNull, value); + } else { + updateBooleanLastValue(groupId, xNull, value, time); + } + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java index bbdb71c28108..80af97d86f01 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java @@ -27,15 +27,20 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.UnSupportedDataTypeException; -public class GroupedMaxAccumulator implements GroupedAccumulator { +public class GroupedMaxAccumulator + implements GroupedAccumulator, + TypeServices.GroupedValueAccessor, + TypeServices.GroupedInputConsumer { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedMaxAccumulator.class); private final TSDataType seriesDataType; @@ -48,188 +53,38 @@ public class GroupedMaxAccumulator implements GroupedAccumulator { private DoubleBigArray doubleValues; private BinaryBigArray binaryValues; private BooleanBigArray booleanValues; + private final TypeServices.GroupedValueService valueService; + private final TypeServices.GroupedInputReader inputReader; public GroupedMaxAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - switch (seriesDataType) { - case INT32: - case DATE: - // Use MIN_VALUE to init NumericType to optimize updateValue method - intValues = new IntBigArray(Integer.MIN_VALUE); - return; - case INT64: - case TIMESTAMP: - longValues = new LongBigArray(Long.MIN_VALUE); - return; - case FLOAT: - floatValues = new FloatBigArray(Float.NEGATIVE_INFINITY); - return; - case DOUBLE: - doubleValues = new DoubleBigArray(Double.NEGATIVE_INFINITY); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - binaryValues = new BinaryBigArray(); - return; - case BOOLEAN: - booleanValues = new BooleanBigArray(); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); - } + Type type = Type.fromTsDataType(seriesDataType); + this.valueService = TypeServices.GROUPED_VALUE_SERVICE.call(type); + this.inputReader = TypeServices.GROUPED_INPUT_READER_SERVICE.call(type); + valueService.initialize(this); } @Override public long getEstimatedSize() { - long valuesSize = 0; - switch (seriesDataType) { - case INT32: - case DATE: - valuesSize += intValues.sizeOf(); - break; - case INT64: - case TIMESTAMP: - valuesSize += longValues.sizeOf(); - break; - case FLOAT: - valuesSize += floatValues.sizeOf(); - break; - case DOUBLE: - valuesSize += doubleValues.sizeOf(); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - valuesSize += binaryValues.sizeOf(); - break; - case BOOLEAN: - valuesSize += booleanValues.sizeOf(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); - } - - return INSTANCE_SIZE + valuesSize + inits.sizeOf(); + return INSTANCE_SIZE + valueService.sizeOf(this) + inits.sizeOf(); } @Override public void setGroupCount(long groupCount) { inits.ensureCapacity(groupCount); - switch (seriesDataType) { - case INT32: - case DATE: - intValues.ensureCapacity(groupCount); - return; - case INT64: - case TIMESTAMP: - longValues.ensureCapacity(groupCount); - return; - case FLOAT: - floatValues.ensureCapacity(groupCount); - return; - case DOUBLE: - doubleValues.ensureCapacity(groupCount); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - binaryValues.ensureCapacity(groupCount); - return; - case BOOLEAN: - booleanValues.ensureCapacity(groupCount); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); - } + valueService.ensureCapacity(this, groupCount); } @Override public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(groupIds, arguments[0], mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(groupIds, arguments[0], mask); - return; - case FLOAT: - addFloatInput(groupIds, arguments[0], mask); - return; - case DOUBLE: - addDoubleInput(groupIds, arguments[0], mask); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(groupIds, arguments[0], mask); - return; - case BOOLEAN: - addBooleanInput(groupIds, arguments[0], mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); - } + inputReader.addInput(groupIds, arguments, mask, this); } @Override public void addIntermediate(int[] groupIds, Column argument) { - for (int i = 0; i < groupIds.length; i++) { - if (argument.isNull(i)) { - continue; - } - - switch (seriesDataType) { - case INT32: - case DATE: - updateIntValue(groupIds[i], argument.getInt(i)); - break; - case INT64: - case TIMESTAMP: - updateLongValue(groupIds[i], argument.getLong(i)); - break; - case FLOAT: - updateFloatValue(groupIds[i], argument.getFloat(i)); - break; - case DOUBLE: - updateDoubleValue(groupIds[i], argument.getDouble(i)); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - updateBinaryValue(groupIds[i], argument.getBinary(i)); - break; - case BOOLEAN: - updateBooleanValue(groupIds[i], argument.getBoolean(i)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); - } - } + inputReader.addIntermediate(groupIds, argument, this); } @Override @@ -238,36 +93,7 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { if (!inits.get(groupId)) { columnBuilder.appendNull(); } else { - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(intValues.get(groupId)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(longValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(floatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(doubleValues.get(groupId)); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - columnBuilder.writeBinary(binaryValues.get(groupId)); - break; - case BOOLEAN: - columnBuilder.writeBoolean(booleanValues.get(groupId)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); - } + valueService.write(this, groupId, columnBuilder); } } @@ -276,36 +102,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { if (!inits.get(groupId)) { columnBuilder.appendNull(); } else { - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(intValues.get(groupId)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(longValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(floatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(doubleValues.get(groupId)); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - columnBuilder.writeBinary(binaryValues.get(groupId)); - break; - case BOOLEAN: - columnBuilder.writeBoolean(booleanValues.get(groupId)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); - } + valueService.write(this, groupId, columnBuilder); } } @@ -315,36 +112,7 @@ public void prepareFinal() {} @Override public void reset() { inits.reset(); - switch (seriesDataType) { - case INT32: - case DATE: - intValues.reset(); - return; - case INT64: - case TIMESTAMP: - longValues.reset(); - return; - case FLOAT: - floatValues.reset(); - return; - case DOUBLE: - doubleValues.reset(); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - binaryValues.reset(); - return; - case BOOLEAN: - booleanValues.reset(); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_AGGREGATION_ARG_CCB09C60, - seriesDataType)); - } + valueService.reset(this); } private void addIntInput(int[] groupIds, Column valueColumn, AggregationMask mask) { @@ -519,4 +287,102 @@ protected void updateBooleanValue(int groupId, boolean value) { booleanValues.set(groupId, value); } } + + @Override + public RuntimeException unsupportedException() { + return new UnSupportedDataTypeException( + String.format(CalcMessages.UNSUPPORTED_DATA_TYPE_IN_MAX_AGGREGATION, seriesDataType)); + } + + @Override + public void initializeIntValues() { + intValues = new IntBigArray(Integer.MIN_VALUE); + } + + @Override + public void initializeLongValues() { + longValues = new LongBigArray(Long.MIN_VALUE); + } + + @Override + public void initializeFloatValues() { + // MIN_VALUE is positive; negative infinity allows all non-NaN values to initialize a group. + floatValues = new FloatBigArray(Float.NEGATIVE_INFINITY); + } + + @Override + public void initializeDoubleValues() { + // MIN_VALUE is positive; negative infinity allows all non-NaN values to initialize a group. + doubleValues = new DoubleBigArray(Double.NEGATIVE_INFINITY); + } + + @Override + public void initializeBinaryValues() { + binaryValues = new BinaryBigArray(); + } + + @Override + public void initializeBooleanValues() { + booleanValues = new BooleanBigArray(); + } + + @Override + public IntBigArray getIntValues() { + return intValues; + } + + @Override + public LongBigArray getLongValues() { + return longValues; + } + + @Override + public FloatBigArray getFloatValues() { + return floatValues; + } + + @Override + public DoubleBigArray getDoubleValues() { + return doubleValues; + } + + @Override + public BinaryBigArray getBinaryValues() { + return binaryValues; + } + + @Override + public BooleanBigArray getBooleanValues() { + return booleanValues; + } + + @Override + public void updateInt(int groupId, int value) { + updateIntValue(groupId, value); + } + + @Override + public void updateLong(int groupId, long value) { + updateLongValue(groupId, value); + } + + @Override + public void updateFloat(int groupId, float value) { + updateFloatValue(groupId, value); + } + + @Override + public void updateDouble(int groupId, double value) { + updateDoubleValue(groupId, value); + } + + @Override + public void updateBinary(int groupId, Binary value) { + updateBinaryValue(groupId, value); + } + + @Override + public void updateBoolean(int groupId, boolean value) { + updateBooleanValue(groupId, value); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java index 18bf03c3dddc..4a87699297e0 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java @@ -27,6 +27,7 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -35,8 +36,8 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -44,19 +45,26 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.tsfile.utils.BytesUtils.boolToBytes; -import static org.apache.tsfile.utils.BytesUtils.doubleToBytes; -import static org.apache.tsfile.utils.BytesUtils.floatToBytes; -import static org.apache.tsfile.utils.BytesUtils.intToBytes; -import static org.apache.tsfile.utils.BytesUtils.longToBytes; /** max(x,y) returns the value of x associated with the maximum value of y over all input values. */ -public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulator { +public abstract class GroupedMaxMinByBaseAccumulator + implements GroupedAccumulator, TypeServices.GroupedMaxMinByValueUpdater { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedAccumulator.class); private final TSDataType xDataType; private final TSDataType yDataType; + private final Type xType; + private final TypeServices.GroupedValueService xValueService; + private final TypeServices.GroupedValueService yValueService; + private final TypeServices.GroupedValueSerializer xValueSerializer; + private final TypeServices.GroupedValueSerializer yValueSerializer; + private final TypeServices.GroupedValueSetter xValueSetter; + private final TypeServices.GroupedMaxMinByInput inputReader; + private final TypeServices.GroupedMaxMinByIntermediateInput intermediateInputReader; + private final TypeServices.GroupedValueAccessor xValueAccessor = new ValueAccessor(true); + private final TypeServices.GroupedValueAccessor yValueAccessor = new ValueAccessor(false); private final BooleanBigArray inits = new BooleanBigArray(); @@ -79,202 +87,131 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat protected GroupedMaxMinByBaseAccumulator(TSDataType xDataType, TSDataType yDataType) { this.xDataType = xDataType; this.yDataType = yDataType; + this.xType = Type.fromTsDataType(xDataType); + Type yType = Type.fromTsDataType(yDataType); + this.xValueService = TypeServices.GROUPED_VALUE_SERVICE.call(xType); + this.yValueService = TypeServices.GROUPED_VALUE_SERVICE.call(yType); + this.xValueSerializer = TypeServices.GROUPED_VALUE_SERIALIZER_SERVICE.call(xType); + this.yValueSerializer = TypeServices.GROUPED_VALUE_SERIALIZER_SERVICE.call(yType); + this.xValueSetter = TypeServices.GROUPED_VALUE_SETTER_SERVICE.call(xType); + this.inputReader = TypeServices.GROUPED_MAX_MIN_BY_INPUT_SERVICE.call(yType); + this.intermediateInputReader = + TypeServices.GROUPED_MAX_MIN_BY_INTERMEDIATE_INPUT_SERVICE.call(yType); + xValueService.initialize(xValueAccessor); + yValueService.initialize(yValueAccessor); + } - switch (xDataType) { - case INT32: - case DATE: - xIntValues = new IntBigArray(); - break; - case INT64: - case TIMESTAMP: - xLongValues = new LongBigArray(); - break; - case FLOAT: - xFloatValues = new FloatBigArray(); - break; - case DOUBLE: - xDoubleValues = new DoubleBigArray(); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - xBinaryValues = new BinaryBigArray(); - break; - case BOOLEAN: - xBooleanValues = new BooleanBigArray(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - xDataType)); - } + public LongBigArray getXLongValues() { + return xLongValues; + } - switch (yDataType) { - case INT32: - case DATE: - yIntValues = new IntBigArray(); - break; - case INT64: - case TIMESTAMP: - yLongValues = new LongBigArray(); - break; - case FLOAT: - yFloatValues = new FloatBigArray(); - break; - case DOUBLE: - yDoubleValues = new DoubleBigArray(); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - yBinaryValues = new BinaryBigArray(); - break; - case BOOLEAN: - yBooleanValues = new BooleanBigArray(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - yDataType)); - } + public void setXLongValues(LongBigArray xLongValues) { + this.xLongValues = xLongValues; } - @Override - public long getEstimatedSize() { - long valuesSize = 0; - switch (xDataType) { - case INT32: - case DATE: - valuesSize += xIntValues.sizeOf(); - break; - case INT64: - case TIMESTAMP: - valuesSize += xLongValues.sizeOf(); - break; - case FLOAT: - valuesSize += xFloatValues.sizeOf(); - break; - case DOUBLE: - valuesSize += xDoubleValues.sizeOf(); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - valuesSize += xBinaryValues.sizeOf(); - break; - case BOOLEAN: - valuesSize += xBooleanValues.sizeOf(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - xDataType)); - } + public IntBigArray getXIntValues() { + return xIntValues; + } - switch (yDataType) { - case INT32: - case DATE: - valuesSize += yIntValues.sizeOf(); - break; - case INT64: - case TIMESTAMP: - valuesSize += yLongValues.sizeOf(); - break; - case FLOAT: - valuesSize += yFloatValues.sizeOf(); - break; - case DOUBLE: - valuesSize += yDoubleValues.sizeOf(); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - valuesSize += yBinaryValues.sizeOf(); - break; - case BOOLEAN: - valuesSize += yBooleanValues.sizeOf(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - xDataType)); - } + public void setXIntValues(IntBigArray xIntValues) { + this.xIntValues = xIntValues; + } + + public FloatBigArray getXFloatValues() { + return xFloatValues; + } + + public void setXFloatValues(FloatBigArray xFloatValues) { + this.xFloatValues = xFloatValues; + } + + public DoubleBigArray getXDoubleValues() { + return xDoubleValues; + } + + public void setXDoubleValues(DoubleBigArray xDoubleValues) { + this.xDoubleValues = xDoubleValues; + } + + public BinaryBigArray getXBinaryValues() { + return xBinaryValues; + } + + public void setXBinaryValues(BinaryBigArray xBinaryValues) { + this.xBinaryValues = xBinaryValues; + } + + public BooleanBigArray getXBooleanValues() { + return xBooleanValues; + } + + public void setXBooleanValues(BooleanBigArray xBooleanValues) { + this.xBooleanValues = xBooleanValues; + } + + public LongBigArray getYLongValues() { + return yLongValues; + } + + public void setYLongValues(LongBigArray yLongValues) { + this.yLongValues = yLongValues; + } + + public IntBigArray getYIntValues() { + return yIntValues; + } + + public void setYIntValues(IntBigArray yIntValues) { + this.yIntValues = yIntValues; + } + + public FloatBigArray getYFloatValues() { + return yFloatValues; + } + + public void setYFloatValues(FloatBigArray yFloatValues) { + this.yFloatValues = yFloatValues; + } + + public DoubleBigArray getYDoubleValues() { + return yDoubleValues; + } + + public void setYDoubleValues(DoubleBigArray yDoubleValues) { + this.yDoubleValues = yDoubleValues; + } - return INSTANCE_SIZE + valuesSize + inits.sizeOf() + xNulls.sizeOf(); + public BinaryBigArray getYBinaryValues() { + return yBinaryValues; + } + + public void setYBinaryValues(BinaryBigArray yBinaryValues) { + this.yBinaryValues = yBinaryValues; + } + + public BooleanBigArray getYBooleanValues() { + return yBooleanValues; + } + + public void setYBooleanValues(BooleanBigArray yBooleanValues) { + this.yBooleanValues = yBooleanValues; + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + + xValueService.sizeOf(xValueAccessor) + + yValueService.sizeOf(yValueAccessor) + + inits.sizeOf() + + xNulls.sizeOf(); } @Override public void setGroupCount(long groupCount) { inits.ensureCapacity(groupCount); xNulls.ensureCapacity(groupCount); - switch (xDataType) { - case INT32: - case DATE: - xIntValues.ensureCapacity(groupCount); - break; - case INT64: - case TIMESTAMP: - xLongValues.ensureCapacity(groupCount); - break; - case FLOAT: - xFloatValues.ensureCapacity(groupCount); - break; - case DOUBLE: - xDoubleValues.ensureCapacity(groupCount); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - xBinaryValues.ensureCapacity(groupCount); - break; - case BOOLEAN: - xBooleanValues.ensureCapacity(groupCount); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - xDataType)); - } - switch (yDataType) { - case INT32: - case DATE: - yIntValues.ensureCapacity(groupCount); - break; - case INT64: - case TIMESTAMP: - yLongValues.ensureCapacity(groupCount); - break; - case FLOAT: - yFloatValues.ensureCapacity(groupCount); - break; - case DOUBLE: - yDoubleValues.ensureCapacity(groupCount); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - yBinaryValues.ensureCapacity(groupCount); - break; - case BOOLEAN: - yBooleanValues.ensureCapacity(groupCount); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - xDataType)); - } + xValueService.ensureCapacity(xValueAccessor, groupCount); + yValueService.ensureCapacity(yValueAccessor, groupCount); } @Override @@ -284,101 +221,13 @@ public void prepareFinal() {} public void reset() { inits.reset(); xNulls.reset(); - switch (xDataType) { - case INT32: - case DATE: - xIntValues.reset(); - break; - case INT64: - case TIMESTAMP: - xLongValues.reset(); - break; - case FLOAT: - xFloatValues.reset(); - break; - case DOUBLE: - xDoubleValues.reset(); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - xBinaryValues.reset(); - break; - case BOOLEAN: - xBooleanValues.reset(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - xDataType)); - } - - switch (yDataType) { - case INT32: - case DATE: - yIntValues.reset(); - break; - case INT64: - case TIMESTAMP: - yLongValues.reset(); - break; - case FLOAT: - yFloatValues.reset(); - break; - case DOUBLE: - yDoubleValues.reset(); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - yBinaryValues.reset(); - break; - case BOOLEAN: - yBooleanValues.reset(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - yDataType)); - } + xValueService.reset(xValueAccessor); + yValueService.reset(yValueAccessor); } @Override public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - switch (yDataType) { - case INT32: - case DATE: - addIntInput(groupIds, arguments, mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(groupIds, arguments, mask); - return; - case FLOAT: - addFloatInput(groupIds, arguments, mask); - return; - case DOUBLE: - addDoubleInput(groupIds, arguments, mask); - return; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - addBinaryInput(groupIds, arguments, mask); - return; - case BOOLEAN: - addBooleanInput(groupIds, arguments, mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - yDataType)); - } + inputReader.add(groupIds, arguments, mask, this); } @Override @@ -423,29 +272,8 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { writeX(groupId, columnBuilder); } - private void addIntInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!arguments[1].isNull(i)) { - updateIntResult(groupIds[i], arguments[1].getInt(i), arguments[0], i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!arguments[1].isNull(position)) { - updateIntResult( - groupIds[position], arguments[1].getInt(position), arguments[0], position); - } - } - } - } - - private void updateIntResult(int groupId, int yValue, Column xColumn, int xIndex) { + @Override + public void updateInt(int groupId, int yValue, Column xColumn, int xIndex) { if (!inits.get(groupId) || check(yValue, yIntValues.get(groupId))) { inits.set(groupId, true); yIntValues.set(groupId, yValue); @@ -453,29 +281,8 @@ private void updateIntResult(int groupId, int yValue, Column xColumn, int xIndex } } - private void addLongInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!arguments[1].isNull(i)) { - updateLongResult(groupIds[i], arguments[1].getLong(i), arguments[0], i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!arguments[1].isNull(position)) { - updateLongResult( - groupIds[position], arguments[1].getLong(position), arguments[0], position); - } - } - } - } - - private void updateLongResult(int groupId, long yValue, Column xColumn, int xIndex) { + @Override + public void updateLong(int groupId, long yValue, Column xColumn, int xIndex) { if (!inits.get(groupId) || check(yValue, yLongValues.get(groupId))) { inits.set(groupId, true); yLongValues.set(groupId, yValue); @@ -483,29 +290,8 @@ private void updateLongResult(int groupId, long yValue, Column xColumn, int xInd } } - private void addFloatInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!arguments[1].isNull(i)) { - updateFloatResult(groupIds[i], arguments[1].getFloat(i), arguments[0], i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!arguments[1].isNull(position)) { - updateFloatResult( - groupIds[position], arguments[1].getFloat(position), arguments[0], position); - } - } - } - } - - private void updateFloatResult(int groupId, float yValue, Column xColumn, int xIndex) { + @Override + public void updateFloat(int groupId, float yValue, Column xColumn, int xIndex) { if (!inits.get(groupId) || check(yValue, yFloatValues.get(groupId))) { inits.set(groupId, true); yFloatValues.set(groupId, yValue); @@ -513,29 +299,8 @@ private void updateFloatResult(int groupId, float yValue, Column xColumn, int xI } } - private void addDoubleInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!arguments[1].isNull(i)) { - updateDoubleResult(groupIds[i], arguments[1].getDouble(i), arguments[0], i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!arguments[1].isNull(position)) { - updateDoubleResult( - groupIds[position], arguments[1].getDouble(position), arguments[0], position); - } - } - } - } - - private void updateDoubleResult(int groupId, double yValue, Column xColumn, int xIndex) { + @Override + public void updateDouble(int groupId, double yValue, Column xColumn, int xIndex) { if (!inits.get(groupId) || check(yValue, yDoubleValues.get(groupId))) { inits.set(groupId, true); yDoubleValues.set(groupId, yValue); @@ -543,29 +308,8 @@ private void updateDoubleResult(int groupId, double yValue, Column xColumn, int } } - private void addBinaryInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!arguments[1].isNull(i)) { - updateBinaryResult(groupIds[i], arguments[1].getBinary(i), arguments[0], i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!arguments[1].isNull(position)) { - updateBinaryResult( - groupIds[position], arguments[1].getBinary(position), arguments[0], position); - } - } - } - } - - private void updateBinaryResult(int groupId, Binary yValue, Column xColumn, int xIndex) { + @Override + public void updateBinary(int groupId, Binary yValue, Column xColumn, int xIndex) { if (!inits.get(groupId) || check(yValue, yBinaryValues.get(groupId))) { inits.set(groupId, true); yBinaryValues.set(groupId, yValue); @@ -573,29 +317,8 @@ private void updateBinaryResult(int groupId, Binary yValue, Column xColumn, int } } - private void addBooleanInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!arguments[1].isNull(i)) { - updateBooleanResult(groupIds[i], arguments[1].getBoolean(i), arguments[0], i); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!arguments[1].isNull(position)) { - updateBooleanResult( - groupIds[position], arguments[1].getBoolean(position), arguments[0], position); - } - } - } - } - - private void updateBooleanResult(int groupId, boolean yValue, Column xColumn, int xIndex) { + @Override + public void updateBoolean(int groupId, boolean yValue, Column xColumn, int xIndex) { if (!inits.get(groupId) || check(yValue, yBooleanValues.get(groupId))) { inits.set(groupId, true); yBooleanValues.set(groupId, yValue); @@ -608,36 +331,7 @@ private void writeX(int groupId, ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (xDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(xIntValues.get(groupId)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(xLongValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(xFloatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(xDoubleValues.get(groupId)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - columnBuilder.writeBinary(xBinaryValues.get(groupId)); - break; - case BOOLEAN: - columnBuilder.writeBoolean(xBooleanValues.get(groupId)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - xDataType)); - } + xValueService.write(xValueAccessor, groupId, columnBuilder); } private void updateX(int groupId, Column xColumn, int xIndex) { @@ -645,239 +339,128 @@ private void updateX(int groupId, Column xColumn, int xIndex) { xNulls.set(groupId, true); } else { xNulls.set(groupId, false); - switch (xDataType) { - case INT32: - case DATE: - xIntValues.set(groupId, xColumn.getInt(xIndex)); - break; - case INT64: - case TIMESTAMP: - xLongValues.set(groupId, xColumn.getLong(xIndex)); - break; - case FLOAT: - xFloatValues.set(groupId, xColumn.getFloat(xIndex)); - break; - case DOUBLE: - xDoubleValues.set(groupId, xColumn.getDouble(xIndex)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - xBinaryValues.set(groupId, xColumn.getBinary(xIndex)); - break; - case BOOLEAN: - xBooleanValues.set(groupId, xColumn.getBoolean(xIndex)); - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - xDataType)); - } + xValueSetter.set(xValueAccessor, groupId, xColumn, xIndex); } } private byte[] serialize(int groupId) { boolean xNull = xNulls.get(groupId); - int yLength = calculateValueLength(groupId, yDataType, false); - int length = yLength + 1 + (xNull ? 0 : calculateValueLength(groupId, xDataType, true)); + int yLength = yValueSerializer.calcTypeSize(yValueAccessor, groupId); + int length = yLength + 1 + (xNull ? 0 : xValueSerializer.calcTypeSize(xValueAccessor, groupId)); byte[] bytes = new byte[length]; - writeIntermediate(groupId, false, yDataType, bytes, 0); + yValueSerializer.serialize(yValueAccessor, groupId, bytes, 0); boolToBytes(xNull, bytes, yLength); if (!xNull) { - writeIntermediate(groupId, true, xDataType, bytes, yLength + 1); + xValueSerializer.serialize(xValueAccessor, groupId, bytes, yLength + 1); } return bytes; } - private void writeIntermediate( - int groupId, boolean isX, TSDataType dataType, byte[] bytes, int offset) { - switch (dataType) { - case INT32: - case DATE: - if (isX) { - intToBytes(xIntValues.get(groupId), bytes, offset); - } else { - intToBytes(yIntValues.get(groupId), bytes, offset); - } - break; - case INT64: - case TIMESTAMP: - if (isX) { - longToBytes(xLongValues.get(groupId), bytes, offset); - } else { - longToBytes(yLongValues.get(groupId), bytes, offset); - } - break; - case FLOAT: - if (isX) { - floatToBytes(xFloatValues.get(groupId), bytes, offset); - } else { - floatToBytes(yFloatValues.get(groupId), bytes, offset); - } - break; - case DOUBLE: - if (isX) { - doubleToBytes(xDoubleValues.get(groupId), bytes, offset); - } else { - doubleToBytes(yDoubleValues.get(groupId), bytes, offset); - } - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - byte[] values = - isX ? xBinaryValues.get(groupId).getValues() : yBinaryValues.get(groupId).getValues(); - intToBytes(values.length, bytes, offset); - System.arraycopy(values, 0, bytes, offset + Integer.BYTES, values.length); - break; - case BOOLEAN: - if (isX) { - boolToBytes(yBooleanValues.get(groupId), bytes, offset); - } else { - boolToBytes(yBooleanValues.get(groupId), bytes, offset); - } - break; - - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - dataType)); - } + private void updateFromBytesIntermediateInput(int groupId, byte[] bytes) { + TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(xDataType)); + ColumnBuilder columnBuilder = builder.getValueColumnBuilders()[0]; + intermediateInputReader.update(groupId, bytes, xType, columnBuilder, this); } - private int calculateValueLength(int groupId, TSDataType dataType, boolean isX) { - switch (dataType) { - case INT32: - case DATE: - return Integer.BYTES; - case INT64: - case TIMESTAMP: - return Long.BYTES; - case FLOAT: - return Float.BYTES; - case DOUBLE: - return Double.BYTES; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - return Integer.BYTES - + (isX - ? xBinaryValues.get(groupId).getValues().length - : yBinaryValues.get(groupId).getValues().length); - case BOOLEAN: - return 1; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - dataType)); + private final class ValueAccessor implements TypeServices.GroupedValueAccessor { + private final boolean x; + + private ValueAccessor(boolean x) { + this.x = x; } - } - private void updateFromBytesIntermediateInput(int groupId, byte[] bytes) { - // long time = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); - int offset = 0; - // Use Column to store x value - TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(xDataType)); - ColumnBuilder columnBuilder = builder.getValueColumnBuilders()[0]; - switch (yDataType) { - case INT32: - case DATE: - int intMaxVal = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateIntResult(groupId, intMaxVal, columnBuilder.build(), 0); - break; - case INT64: - case TIMESTAMP: - long longMaxVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - offset += Long.BYTES; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateLongResult(groupId, longMaxVal, columnBuilder.build(), 0); - break; - case FLOAT: - float floatMaxVal = BytesUtils.bytesToFloat(bytes, offset); - offset += Float.BYTES; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateFloatResult(groupId, floatMaxVal, columnBuilder.build(), 0); - break; - case DOUBLE: - double doubleMaxVal = BytesUtils.bytesToDouble(bytes, offset); - offset += Long.BYTES; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateDoubleResult(groupId, doubleMaxVal, columnBuilder.build(), 0); - break; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - Binary binaryMaxVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - offset += length; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateBinaryResult(groupId, binaryMaxVal, columnBuilder.build(), 0); - break; - case BOOLEAN: - boolean booleanVal = BytesUtils.bytesToBool(bytes, offset); - offset += 1; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateBooleanResult(groupId, booleanVal, columnBuilder.build(), 0); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - yDataType)); + @Override + public RuntimeException unsupportedException() { + TSDataType dataType = x ? xDataType : yDataType; + return new UnSupportedDataTypeException( + String.format(CalcMessages.UNSUPPORTED_DATA_TYPE_IN_MAX_BY_MIN_BY_AGGREGATION, dataType)); } - } - private void readXFromBytesIntermediateInput( - byte[] bytes, int offset, ColumnBuilder columnBuilder) { - boolean isXNull = BytesUtils.bytesToBool(bytes, offset); - offset += 1; - if (isXNull) { - columnBuilder.appendNull(); - } else { - switch (xDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(BytesUtils.bytesToInt(bytes, offset)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(BytesUtils.bytesToLongFromOffset(bytes, 8, offset)); - break; - case FLOAT: - columnBuilder.writeFloat(BytesUtils.bytesToFloat(bytes, offset)); - break; - case DOUBLE: - columnBuilder.writeDouble(BytesUtils.bytesToDouble(bytes, offset)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - columnBuilder.writeBinary(new Binary(BytesUtils.subBytes(bytes, offset, length))); - break; - case BOOLEAN: - columnBuilder.writeBoolean(BytesUtils.bytesToBool(bytes, offset)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MAX_MIN_AGGREGATION_ARG_982BEBD5, - xDataType)); + @Override + public void initializeIntValues() { + if (x) { + xIntValues = new IntBigArray(); + } else { + yIntValues = new IntBigArray(); + } + } + + @Override + public void initializeLongValues() { + if (x) { + xLongValues = new LongBigArray(); + } else { + yLongValues = new LongBigArray(); + } + } + + @Override + public void initializeFloatValues() { + if (x) { + xFloatValues = new FloatBigArray(); + } else { + yFloatValues = new FloatBigArray(); + } + } + + @Override + public void initializeDoubleValues() { + if (x) { + xDoubleValues = new DoubleBigArray(); + } else { + yDoubleValues = new DoubleBigArray(); + } + } + + @Override + public void initializeBinaryValues() { + if (x) { + xBinaryValues = new BinaryBigArray(); + } else { + yBinaryValues = new BinaryBigArray(); } } + + @Override + public void initializeBooleanValues() { + if (x) { + xBooleanValues = new BooleanBigArray(); + } else { + yBooleanValues = new BooleanBigArray(); + } + } + + @Override + public IntBigArray getIntValues() { + return x ? xIntValues : yIntValues; + } + + @Override + public LongBigArray getLongValues() { + return x ? xLongValues : yLongValues; + } + + @Override + public FloatBigArray getFloatValues() { + return x ? xFloatValues : yFloatValues; + } + + @Override + public DoubleBigArray getDoubleValues() { + return x ? xDoubleValues : yDoubleValues; + } + + @Override + public BinaryBigArray getBinaryValues() { + return x ? xBinaryValues : yBinaryValues; + } + + @Override + public BooleanBigArray getBooleanValues() { + return x ? xBooleanValues : yBooleanValues; + } } /** diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java index 87ea0c725a34..ab708678d53a 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java @@ -27,15 +27,20 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.UnSupportedDataTypeException; -public class GroupedMinAccumulator implements GroupedAccumulator { +public class GroupedMinAccumulator + implements GroupedAccumulator, + TypeServices.GroupedValueAccessor, + TypeServices.GroupedInputConsumer { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedMinAccumulator.class); private final TSDataType seriesDataType; @@ -48,188 +53,38 @@ public class GroupedMinAccumulator implements GroupedAccumulator { private DoubleBigArray doubleValues; private BinaryBigArray binaryValues; private BooleanBigArray booleanValues; + private final TypeServices.GroupedValueService valueService; + private final TypeServices.GroupedInputReader inputReader; public GroupedMinAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - switch (seriesDataType) { - case INT32: - case DATE: - // Use MAX_VALUE to init NumericType to optimize updateValue method - intValues = new IntBigArray(Integer.MAX_VALUE); - return; - case INT64: - case TIMESTAMP: - longValues = new LongBigArray(Long.MAX_VALUE); - return; - case FLOAT: - floatValues = new FloatBigArray(Float.MAX_VALUE); - return; - case DOUBLE: - doubleValues = new DoubleBigArray(Double.MAX_VALUE); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - binaryValues = new BinaryBigArray(); - return; - case BOOLEAN: - booleanValues = new BooleanBigArray(); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); - } + this.valueService = + TypeServices.GROUPED_VALUE_SERVICE.call(Type.fromTsDataType(seriesDataType)); + this.inputReader = + TypeServices.GROUPED_INPUT_READER_SERVICE.call(Type.fromTsDataType(seriesDataType)); + valueService.initialize(this); } @Override public long getEstimatedSize() { - long valuesSize = 0; - switch (seriesDataType) { - case INT32: - case DATE: - valuesSize += intValues.sizeOf(); - break; - case INT64: - case TIMESTAMP: - valuesSize += longValues.sizeOf(); - break; - case FLOAT: - valuesSize += floatValues.sizeOf(); - break; - case DOUBLE: - valuesSize += doubleValues.sizeOf(); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - valuesSize += binaryValues.sizeOf(); - break; - case BOOLEAN: - valuesSize += booleanValues.sizeOf(); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); - } - - return INSTANCE_SIZE + valuesSize + inits.sizeOf(); + return INSTANCE_SIZE + valueService.sizeOf(this) + inits.sizeOf(); } @Override public void setGroupCount(long groupCount) { inits.ensureCapacity(groupCount); - switch (seriesDataType) { - case INT32: - case DATE: - intValues.ensureCapacity(groupCount); - return; - case INT64: - case TIMESTAMP: - longValues.ensureCapacity(groupCount); - return; - case FLOAT: - floatValues.ensureCapacity(groupCount); - return; - case DOUBLE: - doubleValues.ensureCapacity(groupCount); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - binaryValues.ensureCapacity(groupCount); - return; - case BOOLEAN: - booleanValues.ensureCapacity(groupCount); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); - } + valueService.ensureCapacity(this, groupCount); } @Override public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(groupIds, arguments[0], mask); - return; - case INT64: - case TIMESTAMP: - addLongInput(groupIds, arguments[0], mask); - return; - case FLOAT: - addFloatInput(groupIds, arguments[0], mask); - return; - case DOUBLE: - addDoubleInput(groupIds, arguments[0], mask); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(groupIds, arguments[0], mask); - return; - case BOOLEAN: - addBooleanInput(groupIds, arguments[0], mask); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); - } + inputReader.addInput(groupIds, arguments, mask, this); } @Override public void addIntermediate(int[] groupIds, Column argument) { - - for (int i = 0; i < groupIds.length; i++) { - if (argument.isNull(i)) { - continue; - } - - switch (seriesDataType) { - case INT32: - case DATE: - updateIntValue(groupIds[i], argument.getInt(i)); - break; - case INT64: - case TIMESTAMP: - updateLongValue(groupIds[i], argument.getLong(i)); - break; - case FLOAT: - updateFloatValue(groupIds[i], argument.getFloat(i)); - break; - case DOUBLE: - updateDoubleValue(groupIds[i], argument.getDouble(i)); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - updateBinaryValue(groupIds[i], argument.getBinary(i)); - break; - case BOOLEAN: - updateBooleanValue(groupIds[i], argument.getBoolean(i)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); - } - } + inputReader.addIntermediate(groupIds, argument, this); } @Override @@ -238,36 +93,7 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { if (!inits.get(groupId)) { columnBuilder.appendNull(); } else { - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(intValues.get(groupId)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(longValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(floatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(doubleValues.get(groupId)); - break; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - columnBuilder.writeBinary(binaryValues.get(groupId)); - break; - case BOOLEAN: - columnBuilder.writeBoolean(booleanValues.get(groupId)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); - } + valueService.write(this, groupId, columnBuilder); } } @@ -276,36 +102,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { if (!inits.get(groupId)) { columnBuilder.appendNull(); } else { - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(intValues.get(groupId)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(longValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(floatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(doubleValues.get(groupId)); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - columnBuilder.writeBinary(binaryValues.get(groupId)); - break; - case BOOLEAN: - columnBuilder.writeBoolean(booleanValues.get(groupId)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); - } + valueService.write(this, groupId, columnBuilder); } } @@ -315,57 +112,7 @@ public void prepareFinal() {} @Override public void reset() { inits.reset(); - switch (seriesDataType) { - case INT32: - case DATE: - intValues.reset(); - return; - case INT64: - case TIMESTAMP: - longValues.reset(); - return; - case FLOAT: - floatValues.reset(); - return; - case DOUBLE: - doubleValues.reset(); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - binaryValues.reset(); - return; - case BOOLEAN: - booleanValues.reset(); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_MIN_AGGREGATION_ARG_CB73B158, - seriesDataType)); - } - } - - private void addIntInput(int[] groupIds, Column valueColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateIntValue(groupIds[i], valueColumn.getInt(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateIntValue(groupIds[position], valueColumn.getInt(position)); - } - } - } + valueService.reset(this); } protected void updateIntValue(int groupId, int value) { @@ -376,27 +123,6 @@ protected void updateIntValue(int groupId, int value) { } } - private void addLongInput(int[] groupIds, Column valueColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateLongValue(groupIds[i], valueColumn.getLong(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateLongValue(groupIds[position], valueColumn.getLong(position)); - } - } - } - } - protected void updateLongValue(int groupId, long value) { long min = longValues.get(groupId); if (value <= min) { @@ -405,27 +131,6 @@ protected void updateLongValue(int groupId, long value) { } } - private void addFloatInput(int[] groupIds, Column valueColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateFloatValue(groupIds[i], valueColumn.getFloat(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateFloatValue(groupIds[position], valueColumn.getFloat(position)); - } - } - } - } - protected void updateFloatValue(int groupId, float value) { float min = floatValues.get(groupId); if (value <= min) { @@ -434,27 +139,6 @@ protected void updateFloatValue(int groupId, float value) { } } - private void addDoubleInput(int[] groupIds, Column valueColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateDoubleValue(groupIds[i], valueColumn.getDouble(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateDoubleValue(groupIds[position], valueColumn.getDouble(position)); - } - } - } - } - protected void updateDoubleValue(int groupId, double value) { double min = doubleValues.get(groupId); if (value <= min) { @@ -463,27 +147,6 @@ protected void updateDoubleValue(int groupId, double value) { } } - private void addBinaryInput(int[] groupIds, Column valueColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBinaryValue(groupIds[i], valueColumn.getBinary(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBinaryValue(groupIds[position], valueColumn.getBinary(position)); - } - } - } - } - protected void updateBinaryValue(int groupId, Binary value) { Binary min = binaryValues.get(groupId); if (!inits.get(groupId) || value.compareTo(min) < 0) { @@ -492,31 +155,106 @@ protected void updateBinaryValue(int groupId, Binary value) { } } - private void addBooleanInput(int[] groupIds, Column valueColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBooleanValue(groupIds[i], valueColumn.getBoolean(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBooleanValue(groupIds[position], valueColumn.getBoolean(position)); - } - } - } - } - protected void updateBooleanValue(int groupId, boolean value) { if (!inits.get(groupId) || !value) { inits.set(groupId, true); booleanValues.set(groupId, value); } } + + @Override + public RuntimeException unsupportedException() { + return new UnSupportedDataTypeException( + String.format(CalcMessages.UNSUPPORTED_DATA_TYPE_IN_MIN_AGGREGATION, seriesDataType)); + } + + @Override + public void initializeIntValues() { + intValues = new IntBigArray(Integer.MAX_VALUE); + } + + @Override + public void initializeLongValues() { + longValues = new LongBigArray(Long.MAX_VALUE); + } + + @Override + public void initializeFloatValues() { + floatValues = new FloatBigArray(Float.MAX_VALUE); + } + + @Override + public void initializeDoubleValues() { + doubleValues = new DoubleBigArray(Double.MAX_VALUE); + } + + @Override + public void initializeBinaryValues() { + binaryValues = new BinaryBigArray(); + } + + @Override + public void initializeBooleanValues() { + booleanValues = new BooleanBigArray(); + } + + @Override + public IntBigArray getIntValues() { + return intValues; + } + + @Override + public LongBigArray getLongValues() { + return longValues; + } + + @Override + public FloatBigArray getFloatValues() { + return floatValues; + } + + @Override + public DoubleBigArray getDoubleValues() { + return doubleValues; + } + + @Override + public BinaryBigArray getBinaryValues() { + return binaryValues; + } + + @Override + public BooleanBigArray getBooleanValues() { + return booleanValues; + } + + @Override + public void updateInt(int groupId, int value) { + updateIntValue(groupId, value); + } + + @Override + public void updateLong(int groupId, long value) { + updateLongValue(groupId, value); + } + + @Override + public void updateFloat(int groupId, float value) { + updateFloatValue(groupId, value); + } + + @Override + public void updateDouble(int groupId, double value) { + updateDoubleValue(groupId, value); + } + + @Override + public void updateBinary(int groupId, Binary value) { + updateBinaryValue(groupId, value); + } + + @Override + public void updateBoolean(int groupId, boolean value) { + updateBooleanValue(groupId, value); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java index a7560ebb2bfa..d9e3ad8798ef 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java @@ -31,20 +31,19 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.Utils.UNSUPPORTED_TYPE_MESSAGE; -import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.Utils.serializeBinaryValue; import static org.apache.tsfile.utils.BytesUtils.bytesToBool; import static org.apache.tsfile.utils.BytesUtils.bytesToLongFromOffset; -import static org.apache.tsfile.utils.TsPrimitiveType.getByType; public class GroupedModeAccumulator implements GroupedAccumulator { @@ -53,6 +52,7 @@ public class GroupedModeAccumulator implements GroupedAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedModeAccumulator.class); private final TSDataType seriesDataType; + private final Type type; private final MapBigArray countMaps = new MapBigArray(); @@ -60,6 +60,7 @@ public class GroupedModeAccumulator implements GroupedAccumulator { public GroupedModeAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; + this.type = Type.fromTsDataType(seriesDataType); } @Override @@ -75,34 +76,7 @@ public void setGroupCount(long groupCount) { @Override public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - switch (seriesDataType) { - case BOOLEAN: - addBooleanInput(groupIds, arguments[0], mask); - break; - case INT32: - case DATE: - addIntInput(groupIds, arguments[0], mask); - break; - case FLOAT: - addFloatInput(groupIds, arguments[0], mask); - break; - case INT64: - case TIMESTAMP: - addLongInput(groupIds, arguments[0], mask); - break; - case DOUBLE: - addDoubleInput(groupIds, arguments[0], mask); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(groupIds, arguments[0], mask); - break; - default: - throw new UnsupportedOperationException( - String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); - } + addInput(groupIds, arguments[0], mask); } @Override @@ -149,34 +123,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { return; } - switch (seriesDataType) { - case BOOLEAN: - columnBuilder.writeBoolean(maxEntry.getKey().getBoolean()); - break; - case INT32: - case DATE: - columnBuilder.writeInt(maxEntry.getKey().getInt()); - break; - case FLOAT: - columnBuilder.writeFloat(maxEntry.getKey().getFloat()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(maxEntry.getKey().getLong()); - break; - case DOUBLE: - columnBuilder.writeDouble(maxEntry.getKey().getDouble()); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - columnBuilder.writeBinary(maxEntry.getKey().getBinary()); - break; - default: - throw new UnsupportedOperationException( - String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); - } + type.write(columnBuilder, maxEntry.getKey()); } @Override @@ -188,121 +135,52 @@ public void reset() { nullCounts.reset(); } + private void addInput(int[] groupIds, Column column, AggregationMask mask) { + int positionCount = mask.getSelectedPositionCount(); + if (mask.isSelectAll()) { + for (int i = 0; i < positionCount; i++) { + addValue(groupIds[i], column, i); + } + } else { + int[] selectedPositions = mask.getSelectedPositions(); + for (int i = 0; i < positionCount; i++) { + int position = selectedPositions[i]; + addValue(groupIds[position], column, position); + } + } + } + + private void addValue(int groupId, Column column, int position) { + HashMap countMap = countMaps.get(groupId); + if (column.isNull(position)) { + nullCounts.increment(groupId); + } else { + countMap.compute( + column.getTsPrimitiveType(position), (key, count) -> count == null ? 1L : count + 1); + checkMapSize(countMap.size()); + } + } + // haveNull | nullCount (optional) | countMap private byte[] serializeCountMap(int groupId) { - byte[] bytes; int offset = 1 + (nullCounts.get(groupId) == 0 ? 0 : Long.BYTES); HashMap countMap = countMaps.get(groupId); - - switch (seriesDataType) { - case BOOLEAN: - bytes = new byte[offset + Integer.BYTES + (1 + Long.BYTES) * countMap.size()]; - BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); - if (nullCounts.get(groupId) != 0) { - BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); - } - BytesUtils.intToBytes(countMap.size(), bytes, offset); - offset += 4; - for (Map.Entry entry : countMap.entrySet()) { - BytesUtils.boolToBytes(entry.getKey().getBoolean(), bytes, offset); - offset += 1; - BytesUtils.longToBytes(entry.getValue(), bytes, offset); - offset += Long.BYTES; - } - break; - case INT32: - case DATE: - bytes = new byte[offset + Integer.BYTES + (Integer.BYTES + Long.BYTES) * countMap.size()]; - BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); - if (nullCounts.get(groupId) != 0) { - BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); - } - BytesUtils.intToBytes(countMap.size(), bytes, offset); - offset += Integer.BYTES; - for (Map.Entry entry : countMap.entrySet()) { - BytesUtils.intToBytes(entry.getKey().getInt(), bytes, offset); - offset += Integer.BYTES; - BytesUtils.longToBytes(entry.getValue(), bytes, offset); - offset += Long.BYTES; - } - break; - case FLOAT: - bytes = new byte[offset + Integer.BYTES + (Float.BYTES + Long.BYTES) * countMap.size()]; - BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); - if (nullCounts.get(groupId) != 0) { - BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); - } - BytesUtils.intToBytes(countMap.size(), bytes, offset); - offset += Integer.BYTES; - for (Map.Entry entry : countMap.entrySet()) { - BytesUtils.floatToBytes(entry.getKey().getFloat(), bytes, offset); - offset += Float.BYTES; - BytesUtils.longToBytes(entry.getValue(), bytes, offset); - offset += Long.BYTES; - } - break; - case INT64: - case TIMESTAMP: - bytes = new byte[offset + Integer.BYTES + (Long.BYTES + Long.BYTES) * countMap.size()]; - BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); - if (nullCounts.get(groupId) != 0) { - BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); - } - BytesUtils.intToBytes(countMap.size(), bytes, offset); - offset += Integer.BYTES; - for (Map.Entry entry : countMap.entrySet()) { - BytesUtils.longToBytes(entry.getKey().getLong(), bytes, offset); - offset += Long.BYTES; - BytesUtils.longToBytes(entry.getValue(), bytes, offset); - offset += Long.BYTES; - } - break; - case DOUBLE: - bytes = new byte[offset + Integer.BYTES + (Double.BYTES + Long.BYTES) * countMap.size()]; - BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); - if (nullCounts.get(groupId) != 0) { - BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); - } - BytesUtils.intToBytes(countMap.size(), bytes, offset); - offset += Integer.BYTES; - for (Map.Entry entry : countMap.entrySet()) { - BytesUtils.doubleToBytes(entry.getKey().getDouble(), bytes, offset); - offset += Double.BYTES; - BytesUtils.longToBytes(entry.getValue(), bytes, offset); - offset += Long.BYTES; - } - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - bytes = - new byte - [offset - + Integer.BYTES - + (Integer.BYTES + Long.BYTES) * countMap.size() - + countMap.keySet().stream() - .mapToInt(key -> key.getBinary().getValues().length) - .sum()]; - BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); - if (nullCounts.get(groupId) != 0) { - BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); - } - BytesUtils.intToBytes(countMap.size(), bytes, offset); - offset += Integer.BYTES; - for (Map.Entry entry : countMap.entrySet()) { - Binary binary = entry.getKey().getBinary(); - serializeBinaryValue(binary, bytes, offset); - offset += (Integer.BYTES + binary.getLength()); - BytesUtils.longToBytes(entry.getValue(), bytes, offset); - offset += Long.BYTES; - } - break; - default: - throw new UnsupportedOperationException( - String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); + int valueSize = countMap.keySet().stream().mapToInt(type::calcTypeSize).sum(); + byte[] bytes = new byte[offset + Integer.BYTES + valueSize + Long.BYTES * countMap.size()]; + BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); + if (nullCounts.get(groupId) != 0) { + BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); + } + BytesUtils.intToBytes(countMap.size(), bytes, offset); + offset += Integer.BYTES; + for (Map.Entry entry : countMap.entrySet()) { + TsPrimitiveType key = entry.getKey(); + int keySize = type.calcTypeSize(key); + type.toBytes(key, bytes, offset); + offset += keySize; + BytesUtils.longToBytes(entry.getValue(), bytes, offset); + offset += Long.BYTES; } - return bytes; } @@ -317,76 +195,11 @@ private void deserializeAndMergeCountMap(int groupId, byte[] bytes) { offset += Integer.BYTES; HashMap countMap = countMaps.get(groupId); - - switch (seriesDataType) { - case BOOLEAN: - for (int i = 0; i < size; i++) { - TsPrimitiveType key = new TsPrimitiveType.TsBoolean(bytesToBool(bytes, offset)); - offset += 1; - long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - offset += Long.BYTES; - countMap.compute(key, (k, v) -> v == null ? count : v + count); - } - break; - case INT32: - case DATE: - for (int i = 0; i < size; i++) { - TsPrimitiveType key = new TsPrimitiveType.TsInt(BytesUtils.bytesToInt(bytes, offset)); - offset += Integer.BYTES; - long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - offset += Long.BYTES; - countMap.compute(key, (k, v) -> v == null ? count : v + count); - } - break; - case FLOAT: - for (int i = 0; i < size; i++) { - TsPrimitiveType key = new TsPrimitiveType.TsFloat(BytesUtils.bytesToFloat(bytes, offset)); - offset += Float.BYTES; - long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - offset += Long.BYTES; - countMap.compute(key, (k, v) -> v == null ? count : v + count); - } - break; - case INT64: - case TIMESTAMP: - for (int i = 0; i < size; i++) { - TsPrimitiveType key = - new TsPrimitiveType.TsLong( - BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset)); - offset += Long.BYTES; - long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - offset += Long.BYTES; - countMap.compute(key, (k, v) -> v == null ? count : v + count); - } - break; - case DOUBLE: - for (int i = 0; i < size; i++) { - TsPrimitiveType key = - new TsPrimitiveType.TsDouble(BytesUtils.bytesToDouble(bytes, offset)); - offset += Double.BYTES; - long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - offset += Long.BYTES; - countMap.compute(key, (k, v) -> v == null ? count : v + count); - } - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - for (int i = 0; i < size; i++) { - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - TsPrimitiveType key = - new TsPrimitiveType.TsBinary(new Binary(BytesUtils.subBytes(bytes, offset, length))); - offset += length; - long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - offset += Long.BYTES; - countMap.compute(key, (k, v) -> v == null ? count : v + count); - } - break; - default: - throw new UnsupportedOperationException( - String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); + ByteBuffer buffer = ByteBuffer.wrap(bytes, offset, bytes.length - offset); + for (int i = 0; i < size; i++) { + TsPrimitiveType key = type.deserialize(buffer); + long count = buffer.getLong(); + countMap.compute(key, (k, v) -> v == null ? count : v + count); } } @@ -398,7 +211,8 @@ private void addBooleanInput(int[] groupIds, Column column, AggregationMask mask if (!column.isNull(i)) { HashMap countMap = countMaps.get(groupIds[i]); countMap.compute( - getByType(seriesDataType, column.getBoolean(i)), (k, v) -> v == null ? 1 : v + 1); + Type.fromTsDataType(seriesDataType).getTsPrimitiveType(column.getBoolean(i)), + (k, v) -> v == null ? 1 : v + 1); checkMapSize(countMap.size()); } else { @@ -413,7 +227,7 @@ private void addBooleanInput(int[] groupIds, Column column, AggregationMask mask if (!column.isNull(position)) { HashMap countMap = countMaps.get(groupIds[position]); countMap.compute( - getByType(seriesDataType, column.getBoolean(position)), + Type.fromTsDataType(seriesDataType).getTsPrimitiveType(column.getBoolean(position)), (k, v) -> v == null ? 1 : v + 1); checkMapSize(countMap.size()); @@ -432,7 +246,8 @@ private void addIntInput(int[] groupIds, Column column, AggregationMask mask) { if (!column.isNull(i)) { HashMap countMap = countMaps.get(groupIds[i]); countMap.compute( - getByType(seriesDataType, column.getInt(i)), (k, v) -> v == null ? 1 : v + 1); + Type.fromTsDataType(seriesDataType).getTsPrimitiveType(column.getInt(i)), + (k, v) -> v == null ? 1 : v + 1); checkMapSize(countMap.size()); } else { @@ -447,7 +262,8 @@ private void addIntInput(int[] groupIds, Column column, AggregationMask mask) { if (!column.isNull(position)) { HashMap countMap = countMaps.get(groupIds[position]); countMap.compute( - getByType(seriesDataType, column.getInt(position)), (k, v) -> v == null ? 1 : v + 1); + Type.fromTsDataType(seriesDataType).getTsPrimitiveType(column.getInt(position)), + (k, v) -> v == null ? 1 : v + 1); checkMapSize(countMap.size()); } else { @@ -465,7 +281,8 @@ private void addFloatInput(int[] groupIds, Column column, AggregationMask mask) if (!column.isNull(i)) { HashMap countMap = countMaps.get(groupIds[i]); countMap.compute( - getByType(seriesDataType, column.getFloat(i)), (k, v) -> v == null ? 1 : v + 1); + Type.fromTsDataType(seriesDataType).getTsPrimitiveType(column.getFloat(i)), + (k, v) -> v == null ? 1 : v + 1); checkMapSize(countMap.size()); } else { @@ -480,7 +297,7 @@ private void addFloatInput(int[] groupIds, Column column, AggregationMask mask) if (!column.isNull(position)) { HashMap countMap = countMaps.get(groupIds[position]); countMap.compute( - getByType(seriesDataType, column.getFloat(position)), + Type.fromTsDataType(seriesDataType).getTsPrimitiveType(column.getFloat(position)), (k, v) -> v == null ? 1 : v + 1); checkMapSize(countMap.size()); @@ -499,7 +316,8 @@ private void addLongInput(int[] groupIds, Column column, AggregationMask mask) { if (!column.isNull(i)) { HashMap countMap = countMaps.get(groupIds[i]); countMap.compute( - getByType(seriesDataType, column.getLong(i)), (k, v) -> v == null ? 1 : v + 1); + Type.fromTsDataType(seriesDataType).getTsPrimitiveType(column.getLong(i)), + (k, v) -> v == null ? 1 : v + 1); checkMapSize(countMap.size()); } else { @@ -514,7 +332,8 @@ private void addLongInput(int[] groupIds, Column column, AggregationMask mask) { if (!column.isNull(position)) { HashMap countMap = countMaps.get(groupIds[position]); countMap.compute( - getByType(seriesDataType, column.getLong(position)), (k, v) -> v == null ? 1 : v + 1); + Type.fromTsDataType(seriesDataType).getTsPrimitiveType(column.getLong(position)), + (k, v) -> v == null ? 1 : v + 1); checkMapSize(countMap.size()); } else { @@ -532,7 +351,8 @@ private void addDoubleInput(int[] groupIds, Column column, AggregationMask mask) if (!column.isNull(i)) { HashMap countMap = countMaps.get(groupIds[i]); countMap.compute( - getByType(seriesDataType, column.getDouble(i)), (k, v) -> v == null ? 1 : v + 1); + Type.fromTsDataType(seriesDataType).getTsPrimitiveType(column.getDouble(i)), + (k, v) -> v == null ? 1 : v + 1); checkMapSize(countMap.size()); } else { @@ -547,7 +367,7 @@ private void addDoubleInput(int[] groupIds, Column column, AggregationMask mask) if (!column.isNull(position)) { HashMap countMap = countMaps.get(groupIds[position]); countMap.compute( - getByType(seriesDataType, column.getDouble(position)), + Type.fromTsDataType(seriesDataType).getTsPrimitiveType(column.getDouble(position)), (k, v) -> v == null ? 1 : v + 1); checkMapSize(countMap.size()); @@ -566,7 +386,8 @@ private void addBinaryInput(int[] groupIds, Column column, AggregationMask mask) if (!column.isNull(i)) { HashMap countMap = countMaps.get(groupIds[i]); countMap.compute( - getByType(seriesDataType, column.getBinary(i)), (k, v) -> v == null ? 1 : v + 1); + Type.fromTsDataType(seriesDataType).getTsPrimitiveType(column.getBinary(i)), + (k, v) -> v == null ? 1 : v + 1); checkMapSize(countMap.size()); } else { @@ -581,7 +402,7 @@ private void addBinaryInput(int[] groupIds, Column column, AggregationMask mask) if (!column.isNull(position)) { HashMap countMap = countMaps.get(groupIds[position]); countMap.compute( - getByType(seriesDataType, column.getBinary(position)), + Type.fromTsDataType(seriesDataType).getTsPrimitiveType(column.getBinary(position)), (k, v) -> v == null ? 1 : v + 1); checkMapSize(countMap.size()); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedPercentileAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedPercentileAccumulator.java index dbc5a89acfb1..149e42b1296d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedPercentileAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedPercentileAccumulator.java @@ -24,11 +24,13 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.PercentileBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -40,6 +42,8 @@ public class GroupedPercentileAccumulator implements GroupedAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedPercentileAccumulator.class); private final TSDataType seriesDataType; + private final TypeServices.ColumnToDoubleConverter valueConverter; + private final TypeServices.NumericResultWriter resultWriter; // percentage is a query-level constant; it is read once from the first input/intermediate and // kept fixed afterwards, so it never gets reset to 0 by a later all-null batch. private double percentage; @@ -52,6 +56,17 @@ public GroupedPercentileAccumulator( TSDataType seriesDataType, MemoryReservationManager memoryReservationManager) { this.seriesDataType = seriesDataType; this.memoryReservationManager = memoryReservationManager; + Type type = Type.fromTsDataType(seriesDataType); + this.valueConverter = + TypeServices.PERCENTILE_NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(type) + .create( + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_PERCENTILE_AGGREGATION, + seriesDataType))); + this.resultWriter = TypeServices.NUMERIC_RESULT_WRITER_SERVICE.call(type); updateMemoryReservation(); } @@ -69,35 +84,14 @@ public void setGroupCount(long groupCount) { public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { if (arguments.length != 2) { throw new SemanticException( - String.format( - CalcMessages.EXCEPTION_PERCENTILE_REQUIRES_2_ARGUMENTS_BUT_GOT_ARG_F3F1882F, - arguments.length)); + String.format(CalcMessages.PERCENTILE_REQUIRES_TWO_ARGUMENTS, arguments.length)); } if (!percentageInitialized) { percentage = arguments[1].getDouble(0); percentageInitialized = true; } - switch (seriesDataType) { - case INT32: - addIntInput(groupIds, arguments, mask); - break; - case INT64: - case TIMESTAMP: - addLongInput(groupIds, arguments, mask); - break; - case FLOAT: - addFloatInput(groupIds, arguments, mask); - break; - case DOUBLE: - addDoubleInput(groupIds, arguments, mask); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_PERCENTILE_AGGREGATION_ARG_8ACED24D, - seriesDataType)); - } + addInput(groupIds, arguments[0], mask); updateMemoryReservation(); } @@ -141,25 +135,25 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (seriesDataType) { - case INT32: - columnBuilder.writeInt((int) result); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong((long) result); - break; - case FLOAT: - columnBuilder.writeFloat((float) result); - break; - case DOUBLE: - columnBuilder.writeDouble(result); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_PERCENTILE_AGGREGATION_ARG_8ACED24D, - seriesDataType)); + resultWriter.write(columnBuilder, result); + } + + private void addInput(int[] groupIds, Column valueColumn, AggregationMask mask) { + int positionCount = mask.getSelectedPositionCount(); + if (mask.isSelectAll()) { + for (int i = 0; i < positionCount; i++) { + if (!valueColumn.isNull(i)) { + array.get(groupIds[i]).addValue(valueConverter.convert(valueColumn, i)); + } + } + } else { + int[] selectedPositions = mask.getSelectedPositions(); + for (int i = 0; i < positionCount; i++) { + int position = selectedPositions[i]; + if (!valueColumn.isNull(position)) { + array.get(groupIds[position]).addValue(valueConverter.convert(valueColumn, position)); + } + } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedRegressionAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedRegressionAccumulator.java index 944307f41a71..c30841f53757 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedRegressionAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedRegressionAccumulator.java @@ -33,7 +33,6 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; @@ -93,8 +92,8 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { if (arguments[0].isNull(i) || arguments[1].isNull(i)) { continue; } - double y = getDoubleValue(arguments[0], i, yDataType); - double x = getDoubleValue(arguments[1], i, xDataType); + double y = getDoubleValue(arguments[0], i); + double x = getDoubleValue(arguments[1], i); update(groupIds[i], x, y); } } else { @@ -104,31 +103,15 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { if (arguments[0].isNull(position) || arguments[1].isNull(position)) { continue; } - double y = getDoubleValue(arguments[0], position, yDataType); - double x = getDoubleValue(arguments[1], position, xDataType); + double y = getDoubleValue(arguments[0], position); + double x = getDoubleValue(arguments[1], position); update(groupIds[position], x, y); } } } - private double getDoubleValue(Column column, int position, TSDataType dataType) { - switch (dataType) { - case INT32: - case DATE: - return column.getInt(position); - case INT64: - case TIMESTAMP: - return column.getLong(position); - case FLOAT: - return column.getFloat(position); - case DOUBLE: - return column.getDouble(position); - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_REGRESSION_AGGREGATION_ARG_7BB08DA2, - dataType)); - } + private double getDoubleValue(Column column, int position) { + return column.getDouble(position); } private void update(int groupId, double x, double y) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java index d01acb8fd5e5..47b4373d01cc 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java @@ -22,10 +22,12 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BooleanBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.DoubleBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -35,11 +37,22 @@ public class GroupedSumAccumulator implements GroupedAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedSumAccumulator.class); private final TSDataType argumentDataType; + private final TypeServices.ColumnToDoubleConverter valueConverter; private final BooleanBigArray initResult = new BooleanBigArray(); private final DoubleBigArray sumValues = new DoubleBigArray(); public GroupedSumAccumulator(TSDataType argumentDataType) { this.argumentDataType = argumentDataType; + Type type = Type.fromTsDataType(argumentDataType); + this.valueConverter = + TypeServices.AGGREGATION_NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(type) + .create( + () -> + new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_SUM_AGGREGATION, + argumentDataType))); } @Override @@ -55,35 +68,8 @@ public void setGroupCount(long groupCount) { @Override public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - checkArgument( - arguments.length == 1, - CalcMessages.EXCEPTION_ARGUMENT_OF_SUM_SHOULD_BE_ONE_COLUMN_D6E636D1); - switch (argumentDataType) { - case INT32: - addIntInput(groupIds, arguments[0], mask); - return; - case INT64: - addLongInput(groupIds, arguments[0], mask); - return; - case FLOAT: - addFloatInput(groupIds, arguments[0], mask); - return; - case DOUBLE: - addDoubleInput(groupIds, arguments[0], mask); - return; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_SUM_AGGREGATION_ARG_92F5A18D, - argumentDataType)); - } + checkArgument(arguments.length == 1, "argument of SUM should be one column"); + addInput(groupIds, arguments[0], mask); } @Override @@ -106,83 +92,14 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { } } - private void addIntInput(int[] groupIds, Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - initResult.set(groupIds[i], true); - sumValues.add(groupIds[i], column.getInt(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - initResult.set(groupIds[position], true); - sumValues.add(groupIds[position], column.getInt(position)); - } - } - } - } - - private void addLongInput(int[] groupIds, Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - initResult.set(groupIds[i], true); - sumValues.add(groupIds[i], column.getLong(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - initResult.set(groupIds[position], true); - sumValues.add(groupIds[position], column.getLong(position)); - } - } - } - } - - private void addFloatInput(int[] groupIds, Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - initResult.set(groupIds[i], true); - sumValues.add(groupIds[i], column.getFloat(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!column.isNull(position)) { - initResult.set(groupIds[position], true); - sumValues.add(groupIds[position], column.getFloat(position)); - } - } - } - } - - private void addDoubleInput(int[] groupIds, Column column, AggregationMask mask) { + private void addInput(int[] groupIds, Column column, AggregationMask mask) { int positionCount = mask.getSelectedPositionCount(); if (mask.isSelectAll()) { for (int i = 0; i < positionCount; i++) { if (!column.isNull(i)) { initResult.set(groupIds[i], true); - sumValues.add(groupIds[i], column.getDouble(i)); + sumValues.add(groupIds[i], valueConverter.convert(column, i)); } } } else { @@ -192,7 +109,7 @@ private void addDoubleInput(int[] groupIds, Column column, AggregationMask mask) position = selectedPositions[i]; if (!column.isNull(position)) { initResult.set(groupIds[position], true); - sumValues.add(groupIds[position], column.getDouble(position)); + sumValues.add(groupIds[position], valueConverter.convert(column, position)); } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java index 9596da3401b7..fa2f36c62abc 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java @@ -24,6 +24,7 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.DoubleBigArray; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -31,6 +32,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -43,6 +45,7 @@ public class GroupedVarianceAccumulator implements GroupedAccumulator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupedVarianceAccumulator.class); private final TSDataType seriesDataType; + private final TypeServices.ColumnToDoubleConverter doubleValueConverter; private final VarianceAccumulator.VarianceType varianceType; private final LongBigArray counts = new LongBigArray(); @@ -52,6 +55,10 @@ public class GroupedVarianceAccumulator implements GroupedAccumulator { public GroupedVarianceAccumulator( TSDataType seriesDataType, VarianceAccumulator.VarianceType varianceType) { this.seriesDataType = seriesDataType; + this.doubleValueConverter = + TypeServices.NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(Type.fromTsDataType(seriesDataType)) + .create(this::unsupportedDataTypeException); this.varianceType = varianceType; } @@ -69,32 +76,8 @@ public void setGroupCount(long groupCount) { @Override public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { - switch (seriesDataType) { - case INT32: - addIntInput(groupIds, arguments[0], mask); - return; - case INT64: - addLongInput(groupIds, arguments[0], mask); - return; - case FLOAT: - addFloatInput(groupIds, arguments[0], mask); - return; - case DOUBLE: - addDoubleInput(groupIds, arguments[0], mask); - return; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case STRING: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_VARIANCE_AGGREGATION_ARG_C641D425, - seriesDataType)); - } + checkInputDataType(); + updateStateByAdd(groupIds, arguments[0], mask); } @Override @@ -194,7 +177,7 @@ public void reset() { m2s.reset(); } - private void addIntInput(int[] groupIds, Column column, AggregationMask mask) { + private void updateStateByAdd(int[] groupIds, Column column, AggregationMask mask) { int positionCount = mask.getSelectedPositionCount(); if (mask.isSelectAll()) { @@ -203,7 +186,7 @@ private void addIntInput(int[] groupIds, Column column, AggregationMask mask) { continue; } - int value = column.getInt(i); + double value = doubleValueConverter.convert(column, i); counts.increment(groupIds[i]); double delta = value - means.get(groupIds[i]); means.add(groupIds[i], delta / counts.get(groupIds[i])); @@ -218,7 +201,7 @@ private void addIntInput(int[] groupIds, Column column, AggregationMask mask) { continue; } - int value = column.getInt(position); + double value = doubleValueConverter.convert(column, position); counts.increment(groupIds[position]); double delta = value - means.get(groupIds[position]); means.add(groupIds[position], delta / counts.get(groupIds[position])); @@ -227,102 +210,14 @@ private void addIntInput(int[] groupIds, Column column, AggregationMask mask) { } } - private void addLongInput(int[] groupIds, Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (column.isNull(i)) { - continue; - } - - long value = column.getLong(i); - counts.increment(groupIds[i]); - double delta = value - means.get(groupIds[i]); - means.add(groupIds[i], delta / counts.get(groupIds[i])); - m2s.add(groupIds[i], delta * (value - means.get(groupIds[i]))); - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (column.isNull(position)) { - continue; - } - - long value = column.getLong(position); - counts.increment(groupIds[position]); - double delta = value - means.get(groupIds[position]); - means.add(groupIds[position], delta / counts.get(groupIds[position])); - m2s.add(groupIds[position], delta * (value - means.get(groupIds[position]))); - } + private void checkInputDataType() { + if (!seriesDataType.isNumeric()) { + throw unsupportedDataTypeException(); } } - private void addFloatInput(int[] groupIds, Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (column.isNull(i)) { - continue; - } - - float value = column.getFloat(i); - counts.increment(groupIds[i]); - double delta = value - means.get(groupIds[i]); - means.add(groupIds[i], delta / counts.get(groupIds[i])); - m2s.add(groupIds[i], delta * (value - means.get(groupIds[i]))); - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (column.isNull(position)) { - continue; - } - - float value = column.getFloat(position); - counts.increment(groupIds[position]); - double delta = value - means.get(groupIds[position]); - means.add(groupIds[position], delta / counts.get(groupIds[position])); - m2s.add(groupIds[position], delta * (value - means.get(groupIds[position]))); - } - } - } - - private void addDoubleInput(int[] groupIds, Column column, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (column.isNull(i)) { - continue; - } - - double value = column.getDouble(i); - counts.increment(groupIds[i]); - double delta = value - means.get(groupIds[i]); - means.add(groupIds[i], delta / counts.get(groupIds[i])); - m2s.add(groupIds[i], delta * (value - means.get(groupIds[i]))); - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (column.isNull(position)) { - continue; - } - - double value = column.getDouble(position); - counts.increment(groupIds[position]); - double delta = value - means.get(groupIds[position]); - means.add(groupIds[position], delta / counts.get(groupIds[position])); - m2s.add(groupIds[position], delta * (value - means.get(groupIds[position]))); - } - } + private UnSupportedDataTypeException unsupportedDataTypeException() { + return new UnSupportedDataTypeException( + String.format(CalcMessages.UNSUPPORTED_DATA_TYPE_IN_VARIANCE_AGGREGATION, seriesDataType)); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/BinaryBigArray.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/BinaryBigArray.java index 3996b1972543..d3d820f8738e 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/BinaryBigArray.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/BinaryBigArray.java @@ -21,6 +21,7 @@ import org.apache.tsfile.utils.Binary; +import static org.apache.tsfile.utils.BytesUtils.intToBytes; import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOfInstance; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfObject; @@ -48,6 +49,12 @@ public Binary get(long index) { return array.get(index); } + public void toBytes(long index, byte[] bytes, int offset) { + Binary value = get(index); + intToBytes(value.getLength(), bytes, offset); + System.arraycopy(value.getValues(), 0, bytes, offset + Integer.BYTES, value.getLength()); + } + /** * Sets the element of this big array at specified index. * diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/BooleanBigArray.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/BooleanBigArray.java index 267b918fad5e..a41f2130b6e1 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/BooleanBigArray.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/BooleanBigArray.java @@ -25,6 +25,7 @@ import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.SEGMENT_SIZE; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.offset; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.segment; +import static org.apache.tsfile.utils.BytesUtils.boolToBytes; import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOf; import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOfInstance; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray; @@ -67,6 +68,10 @@ public boolean get(long index) { return array[segment(index)][offset(index)]; } + public void toBytes(long index, byte[] bytes, int offset) { + boolToBytes(get(index), bytes, offset); + } + /** * Sets the element of this big array at specified index. * diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/DoubleBigArray.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/DoubleBigArray.java index 492774879a8b..9261ce9a1add 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/DoubleBigArray.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/DoubleBigArray.java @@ -25,6 +25,7 @@ import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.SEGMENT_SIZE; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.offset; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.segment; +import static org.apache.tsfile.utils.BytesUtils.doubleToBytes; import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOf; import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOfInstance; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfDoubleArray; @@ -70,6 +71,10 @@ public double get(long index) { return array[segment(index)][offset(index)]; } + public void toBytes(long index, byte[] bytes, int offset) { + doubleToBytes(get(index), bytes, offset); + } + /** * Sets the element of this big array at specified index. * diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/FloatBigArray.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/FloatBigArray.java index d8597140c179..a5c962a133e3 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/FloatBigArray.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/FloatBigArray.java @@ -25,6 +25,7 @@ import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.SEGMENT_SIZE; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.offset; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.segment; +import static org.apache.tsfile.utils.BytesUtils.floatToBytes; import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOf; import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOfInstance; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfFloatArray; @@ -70,6 +71,10 @@ public float get(long index) { return array[segment(index)][offset(index)]; } + public void toBytes(long index, byte[] bytes, int offset) { + floatToBytes(get(index), bytes, offset); + } + /** * Sets the element of this big array at specified index. * diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/IntBigArray.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/IntBigArray.java index 236bd7d55a02..5051b08ef898 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/IntBigArray.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/IntBigArray.java @@ -25,6 +25,7 @@ import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.SEGMENT_SIZE; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.offset; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.segment; +import static org.apache.tsfile.utils.BytesUtils.intToBytes; import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOf; import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOfInstance; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfIntArray; @@ -74,6 +75,10 @@ public int get(long index) { return array[segment(index)][offset(index)]; } + public void toBytes(long index, byte[] bytes, int offset) { + intToBytes(get(index), bytes, offset); + } + /** * Sets the element of this big array at specified index. * diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/LongBigArray.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/LongBigArray.java index 7a65b49218b3..d489248e3f47 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/LongBigArray.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/array/LongBigArray.java @@ -25,6 +25,7 @@ import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.SEGMENT_SIZE; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.offset; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BigArrays.segment; +import static org.apache.tsfile.utils.BytesUtils.longToBytes; import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOf; import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOfInstance; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfLongArray; @@ -70,6 +71,10 @@ public long get(long index) { return array[segment(index)][offset(index)]; } + public void toBytes(long index, byte[] bytes, int offset) { + longToBytes(get(index), bytes, offset); + } + public long[] getSegment(long index) { return array[segment(index)]; } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionValidation.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionValidation.java index 88f9ce627fa3..192b4fc27907 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionValidation.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionValidation.java @@ -20,10 +20,12 @@ package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; public final class RateFunctionValidation { @@ -44,28 +46,18 @@ public static void validateArgumentCount(Column[] arguments, RateFunctionType fu public static double readValue( Column column, int position, TSDataType valueDataType, RateFunctionType functionType) { - double value; - switch (valueDataType) { - case INT32: - value = column.getInt(position); - break; - case INT64: - value = column.getLong(position); - break; - case FLOAT: - value = column.getFloat(position); - break; - case DOUBLE: - value = column.getDouble(position); - break; - default: - throw new SemanticException( - String.format( - CalcMessages - .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_VALUE_TYPE_ARG_9DD7388D, - functionType.getFunctionName(), - valueDataType)); - } + double value = + TypeServices.AGGREGATION_NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(Type.fromTsDataType(valueDataType)) + .create( + () -> + new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_VALUE_TYPE_ARG_9DD7388D, + functionType.getFunctionName(), + valueDataType))) + .convert(column, position); if (!Double.isFinite(value)) { throw new SemanticException( diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/relational/ColumnTransformerBuilder.java index 15397f8c0906..9beba156d656 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/relational/ColumnTransformerBuilder.java @@ -43,12 +43,6 @@ import org.apache.iotdb.calc.transformation.dag.column.leaf.TimeColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.multi.AbstractGreatestLeastColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.multi.CoalesceColumnTransformer; -import org.apache.iotdb.calc.transformation.dag.column.multi.InBinaryMultiColumnTransformer; -import org.apache.iotdb.calc.transformation.dag.column.multi.InBooleanMultiColumnTransformer; -import org.apache.iotdb.calc.transformation.dag.column.multi.InDoubleMultiColumnTransformer; -import org.apache.iotdb.calc.transformation.dag.column.multi.InFloatMultiColumnTransformer; -import org.apache.iotdb.calc.transformation.dag.column.multi.InInt32MultiColumnTransformer; -import org.apache.iotdb.calc.transformation.dag.column.multi.InInt64MultiColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.multi.InMultiColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.multi.LogicalAndMultiColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.multi.LogicalOrMultiColumnTransformer; @@ -146,6 +140,7 @@ import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.UpperColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.factory.CodecStrategiesFactory; import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.factory.NumericCodecStrategiesFactory; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.queryengine.common.SessionInfo; @@ -215,7 +210,6 @@ import org.apache.tsfile.read.common.type.DateType; import org.apache.tsfile.read.common.type.TimestampType; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.read.common.type.TypeEnum; import org.apache.tsfile.utils.Binary; import javax.annotation.Nullable; @@ -224,11 +218,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -1510,7 +1502,7 @@ public ColumnTransformer visitInPredicate(InPredicate node, Context context) { appendIdentityColumnTransformer(node, BOOLEAN, TSDataType.BOOLEAN, context); } else { ColumnTransformer childColumnTransformer = process(node.getValue(), context); - TypeEnum childTypeEnum = childColumnTransformer.getType().getTypeEnum(); + Type childType = childColumnTransformer.getType(); InListExpression inListExpression = (InListExpression) node.getValueList(); List expressionList = inListExpression.getValues(); List values = new ArrayList<>(); @@ -1524,7 +1516,7 @@ public ColumnTransformer visitInPredicate(InPredicate node, Context context) { } } context.cache.put( - node, constructInColumnTransformer(childTypeEnum, valueColumnTransformerList, values)); + node, constructInColumnTransformer(childType, valueColumnTransformerList, values)); } } @@ -1532,111 +1524,10 @@ public ColumnTransformer visitInPredicate(InPredicate node, Context context) { } private static InMultiColumnTransformer constructInColumnTransformer( - TypeEnum childType, - List valueColumnTransformerList, - List values) { - String errorMsg = "\"%s\" cannot be cast to [%s]"; - switch (childType) { - case INT32: - Set intSet = new HashSet<>(); - for (Literal value : values) { - try { - long v = ((LongLiteral) value).getParsedValue(); - if (v <= Integer.MAX_VALUE && v >= Integer.MIN_VALUE) { - intSet.add((int) v); - } - } catch (IllegalArgumentException e) { - throw new SemanticException(String.format(errorMsg, value, childType)); - } - } - return new InInt32MultiColumnTransformer(intSet, valueColumnTransformerList); - case DATE: - Set dateSet = new HashSet<>(); - for (Literal value : values) { - dateSet.add(Integer.parseInt(((GenericLiteral) value).getValue())); - } - return new InInt32MultiColumnTransformer(dateSet, valueColumnTransformerList); - case INT64: - Set longSet = new HashSet<>(); - for (Literal value : values) { - longSet.add(((LongLiteral) value).getParsedValue()); - } - return new InInt64MultiColumnTransformer(longSet, valueColumnTransformerList); - case TIMESTAMP: - Set timestampSet = new HashSet<>(); - for (Literal value : values) { - try { - if (value instanceof LongLiteral) { - timestampSet.add(((LongLiteral) value).getParsedValue()); - } else if (value instanceof DoubleLiteral) { - timestampSet.add((long) ((DoubleLiteral) value).getValue()); - } else if (value instanceof FloatLiteral) { - timestampSet.add((long) ((FloatLiteral) value).getValue()); - } else if (value instanceof GenericLiteral) { - timestampSet.add(Long.parseLong(((GenericLiteral) value).getValue())); - } else { - throw new SemanticException( - CalcMessages - .EXCEPTION_INLIST_LITERAL_TIMESTAMP_CAN_ONLY_LONGLITERAL_DOUBLELITERAL_GENERICLITERAL_CURRENT_A3105E67 - + value.getClass().getSimpleName()); - } - } catch (IllegalArgumentException e) { - throw new SemanticException(String.format(errorMsg, value, childType)); - } - } - return new InInt64MultiColumnTransformer(timestampSet, valueColumnTransformerList); - case FLOAT: - Set floatSet = new HashSet<>(); - for (Literal value : values) { - try { - if (value instanceof FloatLiteral) { - floatSet.add(((FloatLiteral) value).getValue()); - } else { - floatSet.add((float) ((DoubleLiteral) value).getValue()); - } - } catch (IllegalArgumentException e) { - throw new SemanticException(String.format(errorMsg, value, childType)); - } - } - return new InFloatMultiColumnTransformer(floatSet, valueColumnTransformerList); - case DOUBLE: - Set doubleSet = new HashSet<>(); - for (Literal value : values) { - try { - if (value instanceof FloatLiteral) { - doubleSet.add((double) ((FloatLiteral) value).getValue()); - } else { - doubleSet.add(((DoubleLiteral) value).getValue()); - } - } catch (IllegalArgumentException e) { - throw new SemanticException(String.format(errorMsg, value, childType)); - } - } - return new InDoubleMultiColumnTransformer(doubleSet, valueColumnTransformerList); - case BOOLEAN: - Set booleanSet = new HashSet<>(); - for (Literal value : values) { - booleanSet.add(((BooleanLiteral) value).getValue()); - } - return new InBooleanMultiColumnTransformer(booleanSet, valueColumnTransformerList); - case TEXT: - case STRING: - Set stringSet = new HashSet<>(); - for (Literal value : values) { - stringSet.add( - new Binary(((StringLiteral) value).getValue(), TSFileConfig.STRING_CHARSET)); - } - return new InBinaryMultiColumnTransformer(stringSet, valueColumnTransformerList); - case BLOB: - Set binarySet = new HashSet<>(); - for (Literal value : values) { - binarySet.add(new Binary(((BinaryLiteral) value).getValue())); - } - return new InBinaryMultiColumnTransformer(binarySet, valueColumnTransformerList); - default: - throw new UnsupportedOperationException( - CalcMessages.UNSUPPORTED_DATA_TYPE_LOWER + childType); - } + Type childType, List valueColumnTransformerList, List values) { + return TypeServices.IN_MULTI_COLUMN_TRANSFORMER_SERVICE + .call(childType) + .create(valueColumnTransformerList, values); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/CommonOperatorUtils.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/CommonOperatorUtils.java index a332122b2574..463c22969cef 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/CommonOperatorUtils.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/CommonOperatorUtils.java @@ -28,33 +28,13 @@ import org.apache.iotdb.calc.execution.operator.process.fill.filter.MonthIntervalUSFillFilter; import org.apache.iotdb.calc.execution.operator.process.fill.identity.IdentityFill; import org.apache.iotdb.calc.execution.operator.process.fill.identity.IdentityLinearFill; -import org.apache.iotdb.calc.execution.operator.process.fill.linear.DoubleLinearFill; -import org.apache.iotdb.calc.execution.operator.process.fill.linear.FloatLinearFill; -import org.apache.iotdb.calc.execution.operator.process.fill.linear.IntLinearFill; -import org.apache.iotdb.calc.execution.operator.process.fill.linear.LongLinearFill; -import org.apache.iotdb.calc.execution.operator.process.fill.next.BinaryNextFill; -import org.apache.iotdb.calc.execution.operator.process.fill.next.BooleanNextFill; -import org.apache.iotdb.calc.execution.operator.process.fill.next.DoubleNextFill; -import org.apache.iotdb.calc.execution.operator.process.fill.next.FloatNextFill; -import org.apache.iotdb.calc.execution.operator.process.fill.next.IntNextFill; -import org.apache.iotdb.calc.execution.operator.process.fill.next.LongNextFill; -import org.apache.iotdb.calc.execution.operator.process.fill.previous.BinaryPreviousFill; -import org.apache.iotdb.calc.execution.operator.process.fill.previous.BinaryPreviousFillWithTimeDuration; -import org.apache.iotdb.calc.execution.operator.process.fill.previous.BooleanPreviousFill; -import org.apache.iotdb.calc.execution.operator.process.fill.previous.BooleanPreviousFillWithTimeDuration; -import org.apache.iotdb.calc.execution.operator.process.fill.previous.DoublePreviousFill; -import org.apache.iotdb.calc.execution.operator.process.fill.previous.DoublePreviousFillWithTimeDuration; -import org.apache.iotdb.calc.execution.operator.process.fill.previous.FloatPreviousFill; -import org.apache.iotdb.calc.execution.operator.process.fill.previous.FloatPreviousFillWithTimeDuration; -import org.apache.iotdb.calc.execution.operator.process.fill.previous.IntPreviousFill; -import org.apache.iotdb.calc.execution.operator.process.fill.previous.IntPreviousFillWithTimeDuration; -import org.apache.iotdb.calc.execution.operator.process.fill.previous.LongPreviousFill; -import org.apache.iotdb.calc.execution.operator.process.fill.previous.LongPreviousFillWithTimeDuration; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.i18n.QueryMessages; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.column.LongColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.TimeDuration; import java.time.ZoneId; @@ -77,31 +57,8 @@ public class CommonOperatorUtils { public static ILinearFill[] getLinearFill(int inputColumns, List inputDataTypes) { ILinearFill[] linearFill = new ILinearFill[inputColumns]; for (int i = 0; i < inputColumns; i++) { - switch (inputDataTypes.get(i)) { - case INT32: - case DATE: - linearFill[i] = new IntLinearFill(); - break; - case INT64: - case TIMESTAMP: - linearFill[i] = new LongLinearFill(); - break; - case FLOAT: - linearFill[i] = new FloatLinearFill(); - break; - case DOUBLE: - linearFill[i] = new DoubleLinearFill(); - break; - case BOOLEAN: - case TEXT: - case STRING: - case BLOB: - case OBJECT: - linearFill[i] = IDENTITY_LINEAR_FILL; - break; - default: - throw new IllegalArgumentException(UNKNOWN_DATATYPE + inputDataTypes.get(i)); - } + linearFill[i] = + TypeServices.LINEAR_FILL_SERVICE.call(Type.fromTsDataType(inputDataTypes.get(i))).get(); } return linearFill; } @@ -115,49 +72,10 @@ public static IFill[] getPreviousFill( IFill[] previousFill = new IFill[inputColumns]; for (int i = 0; i < inputColumns; i++) { - switch (inputDataTypes.get(i)) { - case BOOLEAN: - previousFill[i] = - filter == null - ? new BooleanPreviousFill() - : new BooleanPreviousFillWithTimeDuration(filter); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - previousFill[i] = - filter == null - ? new BinaryPreviousFill() - : new BinaryPreviousFillWithTimeDuration(filter); - break; - case INT32: - case DATE: - previousFill[i] = - filter == null ? new IntPreviousFill() : new IntPreviousFillWithTimeDuration(filter); - break; - case INT64: - case TIMESTAMP: - previousFill[i] = - filter == null - ? new LongPreviousFill() - : new LongPreviousFillWithTimeDuration(filter); - break; - case FLOAT: - previousFill[i] = - filter == null - ? new FloatPreviousFill() - : new FloatPreviousFillWithTimeDuration(filter); - break; - case DOUBLE: - previousFill[i] = - filter == null - ? new DoublePreviousFill() - : new DoublePreviousFillWithTimeDuration(filter); - break; - default: - throw new IllegalArgumentException(UNKNOWN_DATATYPE + inputDataTypes.get(i)); - } + previousFill[i] = + TypeServices.PREVIOUS_FILL_SERVICE + .call(Type.fromTsDataType(inputDataTypes.get(i))) + .apply(filter); } return previousFill; } @@ -171,33 +89,10 @@ public static ILinearFill[] getNextFill( ILinearFill[] nextFill = new ILinearFill[inputColumns]; for (int i = 0; i < inputColumns; i++) { - switch (inputDataTypes.get(i)) { - case BOOLEAN: - nextFill[i] = new BooleanNextFill(filter); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - nextFill[i] = new BinaryNextFill(filter); - break; - case INT32: - case DATE: - nextFill[i] = new IntNextFill(filter); - break; - case INT64: - case TIMESTAMP: - nextFill[i] = new LongNextFill(filter); - break; - case FLOAT: - nextFill[i] = new FloatNextFill(filter); - break; - case DOUBLE: - nextFill[i] = new DoubleNextFill(filter); - break; - default: - throw new IllegalArgumentException(UNKNOWN_DATATYPE + inputDataTypes.get(i)); - } + nextFill[i] = + TypeServices.NEXT_FILL_SERVICE + .call(Type.fromTsDataType(inputDataTypes.get(i))) + .apply(filter); } return nextFill; } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java index ed52de7958b0..5a4ac7f9ac8b 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java @@ -43,12 +43,6 @@ import org.apache.iotdb.calc.execution.operator.process.ValuesOperator; import org.apache.iotdb.calc.execution.operator.process.fill.IFill; import org.apache.iotdb.calc.execution.operator.process.fill.ILinearFill; -import org.apache.iotdb.calc.execution.operator.process.fill.constant.BinaryConstantFill; -import org.apache.iotdb.calc.execution.operator.process.fill.constant.BooleanConstantFill; -import org.apache.iotdb.calc.execution.operator.process.fill.constant.DoubleConstantFill; -import org.apache.iotdb.calc.execution.operator.process.fill.constant.FloatConstantFill; -import org.apache.iotdb.calc.execution.operator.process.fill.constant.IntConstantFill; -import org.apache.iotdb.calc.execution.operator.process.fill.constant.LongConstantFill; import org.apache.iotdb.calc.execution.operator.process.function.TableFunctionLeafOperator; import org.apache.iotdb.calc.execution.operator.process.function.TableFunctionOperator; import org.apache.iotdb.calc.execution.operator.process.gapfill.GapFillWGroupWMoOperator; @@ -98,17 +92,9 @@ import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.calc.plan.relational.metadata.ITypeMetadata; -import org.apache.iotdb.calc.plan.relational.planner.CastToBlobLiteralVisitor; -import org.apache.iotdb.calc.plan.relational.planner.CastToBooleanLiteralVisitor; -import org.apache.iotdb.calc.plan.relational.planner.CastToDateLiteralVisitor; -import org.apache.iotdb.calc.plan.relational.planner.CastToDoubleLiteralVisitor; -import org.apache.iotdb.calc.plan.relational.planner.CastToFloatLiteralVisitor; -import org.apache.iotdb.calc.plan.relational.planner.CastToInt32LiteralVisitor; -import org.apache.iotdb.calc.plan.relational.planner.CastToInt64LiteralVisitor; -import org.apache.iotdb.calc.plan.relational.planner.CastToStringLiteralVisitor; -import org.apache.iotdb.calc.plan.relational.planner.CastToTimestampLiteralVisitor; import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.calc.utils.datastructure.SortKey; import org.apache.iotdb.common.rpc.thrift.TAggregationType; import org.apache.iotdb.commons.exception.SemanticException; @@ -180,19 +166,11 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ListMultimap; import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; -import org.apache.tsfile.read.common.block.column.BinaryColumn; -import org.apache.tsfile.read.common.block.column.BooleanColumn; -import org.apache.tsfile.read.common.block.column.DoubleColumn; -import org.apache.tsfile.read.common.block.column.FloatColumn; -import org.apache.tsfile.read.common.block.column.IntColumn; -import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.utils.Binary; import java.util.AbstractMap; import java.util.ArrayList; @@ -206,7 +184,6 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.Set; -import java.util.function.BiFunction; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -221,9 +198,7 @@ import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AccumulatorFactory.createAccumulator; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AccumulatorFactory.createBuiltinAccumulator; import static org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AccumulatorFactory.createGroupedAccumulator; -import static org.apache.iotdb.calc.plan.planner.CommonOperatorUtils.IDENTITY_FILL; import static org.apache.iotdb.calc.plan.planner.CommonOperatorUtils.TIME_COLUMN_TEMPLATE; -import static org.apache.iotdb.calc.plan.planner.CommonOperatorUtils.UNKNOWN_DATATYPE; import static org.apache.iotdb.calc.plan.planner.CommonOperatorUtils.getLinearFill; import static org.apache.iotdb.calc.plan.planner.CommonOperatorUtils.getNextFill; import static org.apache.iotdb.calc.plan.planner.CommonOperatorUtils.getPreviousFill; @@ -707,85 +682,10 @@ private IFill[] getValueFill( int inputColumnCount, List inputDataTypes, Literal filledValue, C context) { IFill[] constantFill = new IFill[inputColumnCount]; for (int i = 0; i < inputColumnCount; i++) { - switch (inputDataTypes.get(i)) { - case BOOLEAN: - Boolean bool = filledValue.accept(new CastToBooleanLiteralVisitor(), null); - if (bool == null) { - constantFill[i] = IDENTITY_FILL; - } else { - constantFill[i] = new BooleanConstantFill(bool); - } - break; - case TEXT: - case STRING: - Binary binary = - filledValue.accept(new CastToStringLiteralVisitor(TSFileConfig.STRING_CHARSET), null); - if (binary == null) { - constantFill[i] = IDENTITY_FILL; - } else { - constantFill[i] = new BinaryConstantFill(binary); - } - break; - case BLOB: - Binary blob = filledValue.accept(new CastToBlobLiteralVisitor(), null); - if (blob == null) { - constantFill[i] = IDENTITY_FILL; - } else { - constantFill[i] = new BinaryConstantFill(blob); - } - break; - case INT32: - Integer intValue = filledValue.accept(new CastToInt32LiteralVisitor(), null); - if (intValue == null) { - constantFill[i] = IDENTITY_FILL; - } else { - constantFill[i] = new IntConstantFill(intValue); - } - break; - case DATE: - Integer dateValue = filledValue.accept(new CastToDateLiteralVisitor(), null); - if (dateValue == null) { - constantFill[i] = IDENTITY_FILL; - } else { - constantFill[i] = new IntConstantFill(dateValue); - } - break; - case INT64: - Long longValue = filledValue.accept(new CastToInt64LiteralVisitor(), null); - if (longValue == null) { - constantFill[i] = IDENTITY_FILL; - } else { - constantFill[i] = new LongConstantFill(longValue); - } - break; - case TIMESTAMP: - Long timestampValue = - filledValue.accept(new CastToTimestampLiteralVisitor(context.getZoneId()), null); - if (timestampValue == null) { - constantFill[i] = IDENTITY_FILL; - } else { - constantFill[i] = new LongConstantFill(timestampValue); - } - break; - case FLOAT: - Float floatValue = filledValue.accept(new CastToFloatLiteralVisitor(), null); - if (floatValue == null) { - constantFill[i] = IDENTITY_FILL; - } else { - constantFill[i] = new FloatConstantFill(floatValue); - } - break; - case DOUBLE: - Double doubleValue = filledValue.accept(new CastToDoubleLiteralVisitor(), null); - if (doubleValue == null) { - constantFill[i] = IDENTITY_FILL; - } else { - constantFill[i] = new DoubleConstantFill(doubleValue); - } - break; - default: - throw new IllegalArgumentException(UNKNOWN_DATATYPE + inputDataTypes.get(i)); - } + constantFill[i] = + TypeServices.VALUE_FILL_SERVICE + .call(Type.fromTsDataType(inputDataTypes.get(i))) + .create(filledValue, context.getZoneId()); } return constantFill; } @@ -1207,7 +1107,9 @@ public Operator visitJoin(JoinNode node, C context) { rightOutputSymbolIdx, JoinKeyComparatorFactory.getComparators(joinKeyTypes, true), dataTypes, - joinKeyTypes.stream().map(this::buildUpdateLastRowFunction).collect(Collectors.toList())); + joinKeyTypes.stream() + .map(TypeServices.UPDATE_LAST_ROW_SERVICE::call) + .collect(Collectors.toList())); } else if (requireNonNull(node.getJoinType()) == JoinNode.JoinType.LEFT) { CommonOperatorContext operatorContext = addOperatorContext( @@ -1246,40 +1148,6 @@ protected void semanticCheckForJoin(JoinNode node) { } } - protected BiFunction buildUpdateLastRowFunction(Type joinKeyType) { - switch (joinKeyType.getTypeEnum()) { - case INT32: - return (inputColumn, rowIndex) -> - new IntColumn( - 1, Optional.empty(), new int[] {inputColumn.getInt(rowIndex)}, TSDataType.INT32); - case DATE: - return (inputColumn, rowIndex) -> - new IntColumn( - 1, Optional.empty(), new int[] {inputColumn.getInt(rowIndex)}, TSDataType.DATE); - case INT64: - case TIMESTAMP: - return (inputColumn, rowIndex) -> - new LongColumn(1, Optional.empty(), new long[] {inputColumn.getLong(rowIndex)}); - case FLOAT: - return (inputColumn, rowIndex) -> - new FloatColumn(1, Optional.empty(), new float[] {inputColumn.getFloat(rowIndex)}); - case DOUBLE: - return (inputColumn, rowIndex) -> - new DoubleColumn(1, Optional.empty(), new double[] {inputColumn.getDouble(rowIndex)}); - case BOOLEAN: - return (inputColumn, rowIndex) -> - new BooleanColumn( - 1, Optional.empty(), new boolean[] {inputColumn.getBoolean(rowIndex)}); - case STRING: - case TEXT: - case BLOB: - return (inputColumn, rowIndex) -> - new BinaryColumn(1, Optional.empty(), new Binary[] {inputColumn.getBinary(rowIndex)}); - default: - throw new UnsupportedOperationException(CalcMessages.UNSUPPORTED_DATA_TYPE + joinKeyType); - } - } - @Override public Operator visitSemiJoin(SemiJoinNode node, C context) { List dataTypes = getOutputColumnTypes(node, context.getTableTypeProvider()); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/AbstractGreatestLeastColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/AbstractGreatestLeastColumnTransformer.java index bde0f970e020..2a6f3e909346 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/AbstractGreatestLeastColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/AbstractGreatestLeastColumnTransformer.java @@ -19,13 +19,12 @@ package org.apache.iotdb.calc.transformation.dag.column.multi; -import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.read.common.type.TypeEnum; import java.util.List; @@ -60,52 +59,12 @@ protected void doTransform( public static ColumnTransformer getGreatestColumnTransformer( Type type, List columnTransformers) { - TypeEnum typeEnum = type.getTypeEnum(); - switch (typeEnum) { - case BOOLEAN: - return new BooleanGreatestColumnTransformer(type, columnTransformers); - case INT32: - case DATE: - return new Int32GreatestColumnTransformer(type, columnTransformers); - case INT64: - case TIMESTAMP: - return new Int64GreatestColumnTransformer(type, columnTransformers); - case FLOAT: - return new FloatGreatestColumnTransformer(type, columnTransformers); - case DOUBLE: - return new DoubleGreatestColumnTransformer(type, columnTransformers); - case STRING: - case TEXT: - return new BinaryGreatestColumnTransformer(type, columnTransformers); - default: - throw new UnsupportedOperationException( - String.format(CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_B411C29E, typeEnum)); - } + return TypeServices.GREATEST_COLUMN_TRANSFORMER_SERVICE.call(type).apply(columnTransformers); } public static ColumnTransformer getLeastColumnTransformer( Type type, List columnTransformers) { - TypeEnum typeEnum = type.getTypeEnum(); - switch (typeEnum) { - case BOOLEAN: - return new BooleanLeastColumnTransformer(type, columnTransformers); - case INT32: - case DATE: - return new Int32LeastColumnTransformer(type, columnTransformers); - case INT64: - case TIMESTAMP: - return new Int64LeastColumnTransformer(type, columnTransformers); - case FLOAT: - return new FloatLeastColumnTransformer(type, columnTransformers); - case DOUBLE: - return new DoubleLeastColumnTransformer(type, columnTransformers); - case STRING: - case TEXT: - return new BinaryLeastColumnTransformer(type, columnTransformers); - default: - throw new UnsupportedOperationException( - String.format(CalcMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_B411C29E, typeEnum)); - } + return TypeServices.LEAST_COLUMN_TRANSFORMER_SERVICE.call(type).apply(columnTransformers); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BinaryGreatestColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BinaryGreatestColumnTransformer.java index ae15ca4ee926..45602660afa9 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BinaryGreatestColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BinaryGreatestColumnTransformer.java @@ -29,7 +29,7 @@ import java.util.List; public class BinaryGreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected BinaryGreatestColumnTransformer( + public BinaryGreatestColumnTransformer( Type returnType, List columnTransformerList) { super(returnType, columnTransformerList); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BinaryLeastColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BinaryLeastColumnTransformer.java index 06c43b134d06..2370f258dd4f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BinaryLeastColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BinaryLeastColumnTransformer.java @@ -29,7 +29,7 @@ import java.util.List; public class BinaryLeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected BinaryLeastColumnTransformer( + public BinaryLeastColumnTransformer( Type returnType, List columnTransformerList) { super(returnType, columnTransformerList); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BooleanGreatestColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BooleanGreatestColumnTransformer.java index 89ada0584445..da4320b92772 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BooleanGreatestColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BooleanGreatestColumnTransformer.java @@ -28,7 +28,7 @@ import java.util.List; public class BooleanGreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected BooleanGreatestColumnTransformer( + public BooleanGreatestColumnTransformer( Type returnType, List columnTransformerList) { super(returnType, columnTransformerList); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BooleanLeastColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BooleanLeastColumnTransformer.java index 88054e7cf754..48d57c038c50 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BooleanLeastColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/BooleanLeastColumnTransformer.java @@ -28,7 +28,7 @@ import java.util.List; public class BooleanLeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected BooleanLeastColumnTransformer( + public BooleanLeastColumnTransformer( Type returnType, List columnTransformerList) { super(returnType, columnTransformerList); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/DoubleGreatestColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/DoubleGreatestColumnTransformer.java index cf76de490886..de2711e9d80d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/DoubleGreatestColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/DoubleGreatestColumnTransformer.java @@ -28,7 +28,7 @@ import java.util.List; public class DoubleGreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected DoubleGreatestColumnTransformer( + public DoubleGreatestColumnTransformer( Type returnType, List columnTransformerList) { super(returnType, columnTransformerList); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/DoubleLeastColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/DoubleLeastColumnTransformer.java index c98f124b2926..e5da4489e625 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/DoubleLeastColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/DoubleLeastColumnTransformer.java @@ -28,7 +28,7 @@ import java.util.List; public class DoubleLeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected DoubleLeastColumnTransformer( + public DoubleLeastColumnTransformer( Type returnType, List columnTransformerList) { super(returnType, columnTransformerList); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/FloatGreatestColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/FloatGreatestColumnTransformer.java index ece454c9eb55..87758e75351c 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/FloatGreatestColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/FloatGreatestColumnTransformer.java @@ -28,7 +28,7 @@ import java.util.List; public class FloatGreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected FloatGreatestColumnTransformer( + public FloatGreatestColumnTransformer( Type returnType, List columnTransformerList) { super(returnType, columnTransformerList); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/FloatLeastColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/FloatLeastColumnTransformer.java index 5d7c6155dff9..f06e8c69cb9c 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/FloatLeastColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/FloatLeastColumnTransformer.java @@ -28,7 +28,7 @@ import java.util.List; public class FloatLeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected FloatLeastColumnTransformer( + public FloatLeastColumnTransformer( Type returnType, List columnTransformerList) { super(returnType, columnTransformerList); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int32GreatestColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int32GreatestColumnTransformer.java index ed25278944f4..91df173129de 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int32GreatestColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int32GreatestColumnTransformer.java @@ -28,7 +28,7 @@ import java.util.List; public class Int32GreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected Int32GreatestColumnTransformer( + public Int32GreatestColumnTransformer( Type returnType, List columnTransformerList) { super(returnType, columnTransformerList); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int32LeastColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int32LeastColumnTransformer.java index b33b0022b108..97fc995812d9 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int32LeastColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int32LeastColumnTransformer.java @@ -28,7 +28,7 @@ import java.util.List; public class Int32LeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected Int32LeastColumnTransformer( + public Int32LeastColumnTransformer( Type returnType, List columnTransformerList) { super(returnType, columnTransformerList); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int64GreatestColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int64GreatestColumnTransformer.java index 8f8edb7ab13e..0109ef81f45b 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int64GreatestColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int64GreatestColumnTransformer.java @@ -28,7 +28,7 @@ import java.util.List; public class Int64GreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected Int64GreatestColumnTransformer( + public Int64GreatestColumnTransformer( Type returnType, List columnTransformerList) { super(returnType, columnTransformerList); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int64LeastColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int64LeastColumnTransformer.java index 5c26f6ae92ba..b579b73f4106 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int64LeastColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/Int64LeastColumnTransformer.java @@ -28,7 +28,7 @@ import java.util.List; public class Int64LeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected Int64LeastColumnTransformer( + public Int64LeastColumnTransformer( Type returnType, List columnTransformerList) { super(returnType, columnTransformerList); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/InColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/InColumnTransformer.java index 5b3a6bf37dbb..7e640fd64d55 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/InColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/InColumnTransformer.java @@ -19,32 +19,17 @@ package org.apache.iotdb.calc.transformation.dag.column.unary; -import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; -import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.read.common.type.TypeEnum; -import org.apache.tsfile.utils.Binary; -import java.util.HashSet; import java.util.Set; -import java.util.stream.Collectors; public class InColumnTransformer extends UnaryColumnTransformer { - private final Satisfy satisfy; - - private final TypeEnum childType; - - private Set intSet; - private Set longSet; - private Set floatSet; - private Set doubleSet; - private Set booleanSet; - private Set stringSet; + private final TypeServices.InColumnValueMatcher valueMatcher; public InColumnTransformer( Type returnType, @@ -52,12 +37,15 @@ public InColumnTransformer( boolean isNotIn, Set values) { super(returnType, childColumnTransformer); - satisfy = isNotIn ? new NotInSatisfy() : new InSatisfy(); - this.childType = - childColumnTransformer.getType() == null - ? null - : childColumnTransformer.getType().getTypeEnum(); - initTypedSet(values); + Type childType = childColumnTransformer.getType(); + if (childType == null) { + valueMatcher = (column, position) -> false; + return; + } + TypeServices.InColumnValueMatcher matcher = + TypeServices.IN_COLUMN_VALUE_MATCHER_SERVICE.call(childType).create(values); + this.valueMatcher = + isNotIn ? (column, position) -> !matcher.matches(column, position) : matcher; } @Override @@ -83,190 +71,6 @@ protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] } private void transform(Column column, ColumnBuilder columnBuilder, int i) { - switch (childType) { - case INT32: - case DATE: - returnType.writeBoolean(columnBuilder, satisfy.of(column.getInt(i))); - break; - case INT64: - case TIMESTAMP: - returnType.writeBoolean(columnBuilder, satisfy.of(column.getLong(i))); - break; - case FLOAT: - returnType.writeBoolean(columnBuilder, satisfy.of(column.getFloat(i))); - break; - case DOUBLE: - returnType.writeBoolean(columnBuilder, satisfy.of(column.getDouble(i))); - break; - case BOOLEAN: - returnType.writeBoolean(columnBuilder, satisfy.of(column.getBoolean(i))); - break; - case STRING: - case TEXT: - case BLOB: - returnType.writeBoolean(columnBuilder, satisfy.of(column.getBinary(i))); - break; - default: - throw new UnsupportedOperationException( - CalcMessages.UNSUPPORTED_DATA_TYPE_LOWER + childType); - } - } - - private void initTypedSet(Set values) { - if (childType == null) { - return; - } - String errorMsg = "\"%s\" cannot be cast to [%s]"; - switch (childType) { - case INT32: - intSet = new HashSet<>(); - for (String value : values) { - try { - intSet.add(Integer.valueOf(value)); - } catch (IllegalArgumentException e) { - throw new SemanticException(String.format(errorMsg, value, childType)); - } - } - break; - case INT64: - case TIMESTAMP: - longSet = new HashSet<>(); - for (String value : values) { - try { - longSet.add(Long.valueOf(value)); - } catch (IllegalArgumentException e) { - throw new SemanticException(String.format(errorMsg, value, childType)); - } - } - break; - case FLOAT: - floatSet = new HashSet<>(); - for (String value : values) { - try { - floatSet.add(Float.valueOf(value)); - } catch (IllegalArgumentException e) { - throw new SemanticException(String.format(errorMsg, value, childType)); - } - } - break; - case DOUBLE: - doubleSet = new HashSet<>(); - for (String value : values) { - try { - doubleSet.add(Double.valueOf(value)); - } catch (IllegalArgumentException e) { - throw new SemanticException(String.format(errorMsg, value, childType)); - } - } - break; - case BOOLEAN: - booleanSet = new HashSet<>(); - for (String value : values) { - booleanSet.add(strictCastToBool(value)); - } - break; - case TEXT: - case STRING: - stringSet = - values.stream() - .map(v -> new Binary(v, TSFileConfig.STRING_CHARSET)) - .collect(Collectors.toSet()); - break; - case BLOB: - case DATE: - default: - throw new UnsupportedOperationException( - CalcMessages.UNSUPPORTED_DATA_TYPE_LOWER + childType); - } - } - - private boolean strictCastToBool(String s) { - if ("true".equalsIgnoreCase(s)) { - return true; - } else if ("false".equalsIgnoreCase(s)) { - return false; - } - throw new SemanticException(String.format(CalcMessages.CANNOT_CAST_TO_BOOLEAN, s)); - } - - private interface Satisfy { - - boolean of(int intValue); - - boolean of(long longValue); - - boolean of(float floatValue); - - boolean of(double doubleValue); - - boolean of(boolean booleanValue); - - boolean of(Binary stringValue); - } - - private class InSatisfy implements Satisfy { - - @Override - public boolean of(int intValue) { - return intSet.contains(intValue); - } - - @Override - public boolean of(long longValue) { - return longSet.contains(longValue); - } - - @Override - public boolean of(float floatValue) { - return floatSet.contains(floatValue); - } - - @Override - public boolean of(double doubleValue) { - return doubleSet.contains(doubleValue); - } - - @Override - public boolean of(boolean booleanValue) { - return booleanSet.contains(booleanValue); - } - - @Override - public boolean of(Binary stringValue) { - return stringSet.contains(stringValue); - } - } - - private class NotInSatisfy implements Satisfy { - - @Override - public boolean of(int intValue) { - return !intSet.contains(intValue); - } - - @Override - public boolean of(long longValue) { - return !longSet.contains(longValue); - } - - @Override - public boolean of(float floatValue) { - return !floatSet.contains(floatValue); - } - - @Override - public boolean of(double doubleValue) { - return !doubleSet.contains(doubleValue); - } - - @Override - public boolean of(boolean booleanValue) { - return !booleanSet.contains(booleanValue); - } - - @Override - public boolean of(Binary stringValue) { - return !stringSet.contains(stringValue); - } + returnType.writeBoolean(columnBuilder, valueMatcher.matches(column, i)); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/AbstractCastFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/AbstractCastFunctionColumnTransformer.java index af7a1f600247..caafaef07b8f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/AbstractCastFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/AbstractCastFunctionColumnTransformer.java @@ -19,46 +19,35 @@ package org.apache.iotdb.calc.transformation.dag.column.unary.scalar; -import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.unary.UnaryColumnTransformer; -import org.apache.iotdb.calc.transformation.dag.util.CastFunctionUtils; -import org.apache.iotdb.commons.exception.IoTDBRuntimeException; -import org.apache.iotdb.commons.exception.SemanticException; -import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.read.common.type.TypeEnum; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BytesUtils; -import org.apache.tsfile.utils.DateUtils; import java.time.ZoneId; -import java.time.format.DateTimeParseException; - -import static org.apache.iotdb.calc.transformation.dag.util.CastFunctionUtils.ERROR_MSG; -import static org.apache.iotdb.rpc.TSStatusCode.DATE_OUT_OF_RANGE; public abstract class AbstractCastFunctionColumnTransformer extends UnaryColumnTransformer { private final ZoneId zoneId; + private final TypeServices.CastInputService castInputService; + private final TypeServices.CastValueService castValueService; protected AbstractCastFunctionColumnTransformer( Type returnType, ColumnTransformer childColumnTransformer, ZoneId zoneId) { super(returnType, childColumnTransformer); this.zoneId = zoneId; + this.castInputService = TypeServices.CAST_INPUT_SERVICE.call(childColumnTransformer.getType()); + this.castValueService = TypeServices.CAST_VALUE_SERVICE.call(returnType); } @Override protected void doTransform(Column column, ColumnBuilder columnBuilder) { - Type childType = childColumnTransformer.getType(); - TypeEnum sourceType = childType.getTypeEnum(); for (int i = 0, n = column.getPositionCount(); i < n; i++) { if (!column.isNull(i)) { - transform(column, columnBuilder, sourceType, childType, i); + transform(column, columnBuilder, i); } else { columnBuilder.appendNull(); } @@ -67,349 +56,18 @@ protected void doTransform(Column column, ColumnBuilder columnBuilder) { @Override protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) { - Type childType = childColumnTransformer.getType(); - TypeEnum sourceType = childType.getTypeEnum(); for (int i = 0, n = column.getPositionCount(); i < n; i++) { if (selection[i] && !column.isNull(i)) { - transform(column, columnBuilder, sourceType, childType, i); + transform(column, columnBuilder, i); } else { columnBuilder.appendNull(); } } } - protected abstract void transform( - Column column, ColumnBuilder columnBuilder, TypeEnum sourceType, Type childType, int i); - - protected void cast(ColumnBuilder columnBuilder, int value) { - switch (returnType.getTypeEnum()) { - case INT32: - case DATE: - returnType.writeInt(columnBuilder, value); - break; - case INT64: - case TIMESTAMP: - returnType.writeLong(columnBuilder, value); - break; - case FLOAT: - returnType.writeFloat(columnBuilder, value); - break; - case DOUBLE: - returnType.writeDouble(columnBuilder, value); - break; - case BOOLEAN: - returnType.writeBoolean(columnBuilder, value != 0); - break; - case TEXT: - case STRING: - returnType.writeBinary(columnBuilder, BytesUtils.valueOf(String.valueOf(value))); - break; - case BLOB: - returnType.writeBinary(columnBuilder, new Binary(BytesUtils.intToBytes(value))); - break; - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, returnType.getTypeEnum())); - } - } - - protected void castDate(ColumnBuilder columnBuilder, int value) { - switch (returnType.getTypeEnum()) { - case INT32: - case DATE: - returnType.writeInt(columnBuilder, value); - break; - case INT64: - returnType.writeLong(columnBuilder, value); - break; - case TIMESTAMP: - returnType.writeLong( - columnBuilder, - DateTimeUtils.correctPrecision(DateUtils.parseIntToTimestamp(value, zoneId))); - break; - case FLOAT: - returnType.writeFloat(columnBuilder, value); - break; - case DOUBLE: - returnType.writeDouble(columnBuilder, value); - break; - case BOOLEAN: - returnType.writeBoolean(columnBuilder, value != 0); - break; - case TEXT: - case STRING: - returnType.writeBinary(columnBuilder, BytesUtils.valueOf(DateUtils.formatDate(value))); - break; - case BLOB: - returnType.writeBinary(columnBuilder, new Binary(BytesUtils.intToBytes(value))); - break; - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, returnType.getTypeEnum())); - } - } - - protected void castTimestamp(ColumnBuilder columnBuilder, long value) { - try { - switch (returnType.getTypeEnum()) { - case INT32: - returnType.writeInt(columnBuilder, (CastFunctionUtils.castLongToInt(value))); - break; - case DATE: - returnType.writeInt( - columnBuilder, - DateUtils.parseDateExpressionToInt(DateTimeUtils.convertToLocalDate(value, zoneId))); - break; - case INT64: - case TIMESTAMP: - returnType.writeLong(columnBuilder, value); - break; - case FLOAT: - returnType.writeFloat(columnBuilder, value); - break; - case DOUBLE: - returnType.writeDouble(columnBuilder, value); - break; - case BOOLEAN: - returnType.writeBoolean(columnBuilder, value != 0L); - break; - case TEXT: - case STRING: - returnType.writeBinary( - columnBuilder, BytesUtils.valueOf(DateTimeUtils.convertLongToDate(value, zoneId))); - break; - case BLOB: - returnType.writeBinary(columnBuilder, new Binary(BytesUtils.longToBytes(value))); - break; - default: - throw new UnsupportedOperationException( - String.format(ERROR_MSG, returnType.getTypeEnum())); - } - } catch (DateTimeParseException e) { - throw new IoTDBRuntimeException( - CalcMessages.EXCEPTION_YEAR_MUST_BETWEEN_1000_9999_8FBB94AA, - DATE_OUT_OF_RANGE.getStatusCode(), - true); - } - } - - protected void cast(ColumnBuilder columnBuilder, long value) { - switch (returnType.getTypeEnum()) { - case INT32: - case DATE: - returnType.writeInt(columnBuilder, (CastFunctionUtils.castLongToInt(value))); - break; - case INT64: - case TIMESTAMP: - returnType.writeLong(columnBuilder, value); - break; - case FLOAT: - returnType.writeFloat(columnBuilder, value); - break; - case DOUBLE: - returnType.writeDouble(columnBuilder, value); - break; - case BOOLEAN: - returnType.writeBoolean(columnBuilder, value != 0L); - break; - case TEXT: - case STRING: - returnType.writeBinary(columnBuilder, BytesUtils.valueOf(String.valueOf(value))); - break; - case BLOB: - returnType.writeBinary(columnBuilder, new Binary(BytesUtils.longToBytes(value))); - break; - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, returnType.getTypeEnum())); - } - } - - protected void cast(ColumnBuilder columnBuilder, float value) { - switch (returnType.getTypeEnum()) { - case INT32: - case DATE: - returnType.writeInt(columnBuilder, CastFunctionUtils.castFloatToInt(value)); - break; - case INT64: - case TIMESTAMP: - returnType.writeLong(columnBuilder, CastFunctionUtils.castFloatToLong(value)); - break; - case FLOAT: - returnType.writeFloat(columnBuilder, value); - break; - case DOUBLE: - returnType.writeDouble(columnBuilder, value); - break; - case BOOLEAN: - returnType.writeBoolean(columnBuilder, value != 0.0f); - break; - case TEXT: - case STRING: - returnType.writeBinary(columnBuilder, BytesUtils.valueOf(String.valueOf(value))); - break; - case BLOB: - returnType.writeBinary(columnBuilder, new Binary(BytesUtils.floatToBytes(value))); - break; - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, returnType.getTypeEnum())); - } - } - - protected void cast(ColumnBuilder columnBuilder, double value) { - switch (returnType.getTypeEnum()) { - case INT32: - case DATE: - returnType.writeInt(columnBuilder, CastFunctionUtils.castDoubleToInt(value)); - break; - case INT64: - case TIMESTAMP: - returnType.writeLong(columnBuilder, CastFunctionUtils.castDoubleToLong(value)); - break; - case FLOAT: - returnType.writeFloat(columnBuilder, CastFunctionUtils.castDoubleToFloat(value)); - break; - case DOUBLE: - returnType.writeDouble(columnBuilder, value); - break; - case BOOLEAN: - returnType.writeBoolean(columnBuilder, value != 0.0); - break; - case TEXT: - case STRING: - returnType.writeBinary(columnBuilder, BytesUtils.valueOf(String.valueOf(value))); - break; - case BLOB: - returnType.writeBinary(columnBuilder, new Binary(BytesUtils.doubleToBytes(value))); - break; - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, returnType.getTypeEnum())); - } - } - - protected void cast(ColumnBuilder columnBuilder, boolean value) { - switch (returnType.getTypeEnum()) { - case INT32: - case DATE: - returnType.writeInt(columnBuilder, value ? 1 : 0); - break; - case INT64: - case TIMESTAMP: - returnType.writeLong(columnBuilder, value ? 1L : 0); - break; - case FLOAT: - returnType.writeFloat(columnBuilder, value ? 1.0f : 0); - break; - case DOUBLE: - returnType.writeDouble(columnBuilder, value ? 1.0 : 0); - break; - case BOOLEAN: - returnType.writeBoolean(columnBuilder, value); - break; - case TEXT: - case STRING: - returnType.writeBinary(columnBuilder, BytesUtils.valueOf(String.valueOf(value))); - break; - case BLOB: - returnType.writeBinary(columnBuilder, new Binary(BytesUtils.boolToBytes(value))); - break; - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, returnType.getTypeEnum())); - } - } - - protected void castString(ColumnBuilder columnBuilder, Binary value) { - String stringValue = value.getStringValue(TSFileConfig.STRING_CHARSET); - try { - switch (returnType.getTypeEnum()) { - case INT32: - returnType.writeInt(columnBuilder, Integer.parseInt(stringValue)); - break; - case DATE: - returnType.writeInt(columnBuilder, DateUtils.parseDateExpressionToInt(stringValue)); - break; - case INT64: - returnType.writeLong(columnBuilder, Long.parseLong(stringValue)); - break; - case TIMESTAMP: - returnType.writeLong( - columnBuilder, DateTimeUtils.convertDatetimeStrToLong(stringValue, zoneId)); - break; - case FLOAT: - returnType.writeFloat(columnBuilder, CastFunctionUtils.castTextToFloat(stringValue)); - break; - case DOUBLE: - returnType.writeDouble(columnBuilder, CastFunctionUtils.castTextToDouble(stringValue)); - break; - case BOOLEAN: - returnType.writeBoolean(columnBuilder, CastFunctionUtils.castTextToBoolean(stringValue)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - returnType.writeBinary(columnBuilder, value); - break; - default: - throw new UnsupportedOperationException( - String.format(ERROR_MSG, returnType.getTypeEnum())); - } - } catch (DateTimeParseException | NumberFormatException e) { - throw new SemanticException( - String.format( - CalcMessages.EXCEPTION_CANNOT_CAST_ARG_ARG_TYPE_8266A2C6, - stringValue, - returnType.getDisplayName())); - } - } - - protected void castBlob(ColumnBuilder columnBuilder, Binary value) { - String stringValue = BytesUtils.parseBlobByteArrayToString(value.getValues()); - try { - switch (returnType.getTypeEnum()) { - case INT32: - returnType.writeInt(columnBuilder, Integer.parseInt(stringValue)); - break; - case DATE: - returnType.writeInt(columnBuilder, DateUtils.parseDateExpressionToInt(stringValue)); - break; - case INT64: - returnType.writeLong(columnBuilder, Long.parseLong(stringValue)); - break; - case TIMESTAMP: - returnType.writeLong( - columnBuilder, DateTimeUtils.convertDatetimeStrToLong(stringValue, zoneId)); - break; - case FLOAT: - returnType.writeFloat(columnBuilder, CastFunctionUtils.castTextToFloat(stringValue)); - break; - case DOUBLE: - returnType.writeDouble(columnBuilder, CastFunctionUtils.castTextToDouble(stringValue)); - break; - case BOOLEAN: - returnType.writeBoolean(columnBuilder, CastFunctionUtils.castTextToBoolean(stringValue)); - break; - case TEXT: - case STRING: - case BLOB: - returnType.writeBinary(columnBuilder, value); - break; - default: - throw new UnsupportedOperationException( - String.format(ERROR_MSG, returnType.getTypeEnum())); - } - } catch (DateTimeParseException | NumberFormatException e) { - throw new SemanticException( - String.format( - CalcMessages.EXCEPTION_CANNOT_CAST_ARG_ARG_TYPE_8266A2C6, - stringValue, - returnType.getDisplayName())); - } - } + protected abstract void transform(Column column, ColumnBuilder columnBuilder, int i); - protected void castObject(ColumnBuilder columnBuilder, Binary value) { - String stringValue = BytesUtils.parseObjectByteArrayToString(value.getValues()); - if (returnType.getTypeEnum() == TypeEnum.STRING) { - returnType.writeBinary(columnBuilder, BytesUtils.valueOf(String.valueOf(stringValue))); - } else { - throw new UnsupportedOperationException(String.format(ERROR_MSG, returnType.getTypeEnum())); - } + protected void castSourceValue(Column column, ColumnBuilder columnBuilder, int position) { + castInputService.cast(column, columnBuilder, position, castValueService, zoneId); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/CastFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/CastFunctionColumnTransformer.java index 710117863c35..4882dbb86537 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/CastFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/CastFunctionColumnTransformer.java @@ -19,13 +19,11 @@ package org.apache.iotdb.calc.transformation.dag.column.unary.scalar; -import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.read.common.type.TypeEnum; import java.time.ZoneId; @@ -37,45 +35,7 @@ public CastFunctionColumnTransformer( } @Override - protected void transform( - Column column, ColumnBuilder columnBuilder, TypeEnum sourceType, Type childType, int i) { - switch (sourceType) { - case INT32: - cast(columnBuilder, childType.getInt(column, i)); - break; - case DATE: - castDate(columnBuilder, childType.getInt(column, i)); - break; - case INT64: - cast(columnBuilder, childType.getLong(column, i)); - break; - case TIMESTAMP: - castTimestamp(columnBuilder, childType.getLong(column, i)); - break; - case FLOAT: - cast(columnBuilder, childType.getFloat(column, i)); - break; - case DOUBLE: - cast(columnBuilder, childType.getDouble(column, i)); - break; - case BOOLEAN: - cast(columnBuilder, childType.getBoolean(column, i)); - break; - case TEXT: - case STRING: - castString(columnBuilder, childType.getBinary(column, i)); - break; - case BLOB: - castBlob(columnBuilder, childType.getBinary(column, i)); - break; - case OBJECT: - castObject(columnBuilder, childType.getBinary(column, i)); - break; - default: - throw new UnsupportedOperationException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_SOURCE_DATATYPE_ARG_678B759C, - childColumnTransformer.getType().getTypeEnum())); - } + protected void transform(Column column, ColumnBuilder columnBuilder, int i) { + castSourceValue(column, columnBuilder, i); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/FormatColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/FormatColumnTransformer.java index 3a1a2153b7ae..244c8717dd7c 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/FormatColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/FormatColumnTransformer.java @@ -22,15 +22,13 @@ import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.multi.MultiColumnTransformer; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.exception.SemanticException; -import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.read.common.type.TypeEnum; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.DateUtils; import java.time.ZoneId; import java.util.IllegalFormatConversionException; @@ -43,11 +41,18 @@ public class FormatColumnTransformer extends MultiColumnTransformer { private final ZoneId zoneId; + private final TypeServices.FormatValueConverter[] valueConverters; public FormatColumnTransformer( Type returnType, List columnTransformerList, ZoneId zoneId) { super(returnType, columnTransformerList); this.zoneId = zoneId; + this.valueConverters = new TypeServices.FormatValueConverter[columnTransformerList.size() - 1]; + for (int i = 0; i < valueConverters.length; i++) { + valueConverters[i] = + TypeServices.FORMAT_VALUE_CONVERTER_SERVICE.call( + columnTransformerList.get(i + 1).getType()); + } } @Override @@ -76,11 +81,10 @@ private void transform(List childrenColumns, ColumnBuilder builder, int String pattern = String.valueOf(childrenColumns.get(0).getBinary(i)); for (int j = 0; j < valueColumns.size(); j++) { Column column = valueColumns.get(j); - TypeEnum type = columnTransformerList.get(j + 1).getType().getTypeEnum(); if (column.isNull(i)) { values[j] = null; } else { - values[j] = valueConverter(type, column, i); + values[j] = valueConverters[j].convert(column, i, zoneId); } } try { @@ -94,29 +98,6 @@ private void transform(List childrenColumns, ColumnBuilder builder, int } } - private Object valueConverter(TypeEnum type, Column column, int i) { - switch (type) { - case UNKNOWN: - return null; - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - case BOOLEAN: - case TEXT: - case STRING: - case BLOB: - return column.getObject(i); - case DATE: - return DateUtils.parseIntToLocalDate(column.getInt(i)); - case TIMESTAMP: - return DateTimeUtils.convertToZonedDateTime(column.getLong(i), zoneId); - default: - throw new UnsupportedOperationException( - String.format(CalcMessages.EXCEPTION_UNSUPPORTED_SOURCE_DATATYPE_ARG_678B759C, type)); - } - } - @Override protected void checkType() { // do nothing because the type is checked in tableMetaDataImpl diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/RoundColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/RoundColumnTransformer.java index 4b9ff1486eec..5a1d8a002259 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/RoundColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/RoundColumnTransformer.java @@ -26,7 +26,6 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.read.common.type.TypeEnum; public class RoundColumnTransformer extends BinaryColumnTransformer { @@ -38,7 +37,7 @@ public RoundColumnTransformer( @Override protected void doTransform( Column leftColumn, Column rightColumn, ColumnBuilder builder, int positionCount) { - TypeEnum sourceType = leftTransformer.getType().getTypeEnum(); + Type sourceType = leftTransformer.getType(); for (int i = 0; i < positionCount; i++) { if (!leftColumn.isNull(i) && !rightColumn.isNull(i)) { transform(sourceType, leftColumn, rightColumn, builder, i); @@ -55,7 +54,7 @@ protected void doTransform( ColumnBuilder builder, int positionCount, boolean[] selection) { - TypeEnum sourceType = leftTransformer.getType().getTypeEnum(); + Type sourceType = leftTransformer.getType(); for (int i = 0; i < positionCount; i++) { if (selection[i] && !leftColumn.isNull(i) && !rightColumn.isNull(i)) { transform(sourceType, leftColumn, rightColumn, builder, i); @@ -66,36 +65,10 @@ protected void doTransform( } private void transform( - TypeEnum sourceType, Column leftColumn, Column rightColumn, ColumnBuilder builder, int i) { + Type sourceType, Column leftColumn, Column rightColumn, ColumnBuilder builder, int i) { int places = rightTransformer.getType().getInt(rightColumn, i); - switch (sourceType) { - case INT32: - builder.writeDouble( - Math.rint(leftColumn.getInt(i) * Math.pow(10, places)) / Math.pow(10, places)); - break; - case INT64: - builder.writeDouble( - Math.rint(leftColumn.getLong(i) * Math.pow(10, places)) / Math.pow(10, places)); - break; - case FLOAT: - builder.writeDouble( - Math.rint(leftColumn.getFloat(i) * Math.pow(10, places)) / Math.pow(10, places)); - break; - case DOUBLE: - builder.writeDouble( - Math.rint(leftColumn.getDouble(i) * Math.pow(10, places)) / Math.pow(10, places)); - break; - case DATE: - case TEXT: - case BOOLEAN: - case BLOB: - case STRING: - case TIMESTAMP: - default: - throw new UnsupportedOperationException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_SOURCE_DATATYPE_ARG_678B759C, sourceType)); - } + double scale = Math.pow(10, places); + builder.writeDouble(Math.rint(sourceType.getDouble(leftColumn, i) * scale) / scale); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/RoundFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/RoundFunctionColumnTransformer.java index b10822cfd79d..8104c4e4c0ac 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/RoundFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/RoundFunctionColumnTransformer.java @@ -19,14 +19,12 @@ package org.apache.iotdb.calc.transformation.dag.column.unary.scalar; -import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.unary.UnaryColumnTransformer; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.read.common.type.TypeEnum; public class RoundFunctionColumnTransformer extends UnaryColumnTransformer { @@ -40,7 +38,7 @@ public RoundFunctionColumnTransformer( @Override protected void doTransform(Column column, ColumnBuilder columnBuilder) { - TypeEnum sourceType = childColumnTransformer.getType().getTypeEnum(); + Type sourceType = childColumnTransformer.getType(); for (int i = 0, n = column.getPositionCount(); i < n; i++) { if (!column.isNull(i)) { transform(column, columnBuilder, sourceType, i); @@ -52,7 +50,7 @@ protected void doTransform(Column column, ColumnBuilder columnBuilder) { @Override protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) { - TypeEnum sourceType = childColumnTransformer.getType().getTypeEnum(); + Type sourceType = childColumnTransformer.getType(); for (int i = 0, n = column.getPositionCount(); i < n; i++) { if (selection[i] && !column.isNull(i)) { transform(column, columnBuilder, sourceType, i); @@ -62,35 +60,8 @@ protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] } } - private void transform(Column column, ColumnBuilder columnBuilder, TypeEnum sourceType, int i) { - switch (sourceType) { - case INT32: - columnBuilder.writeDouble( - Math.rint(column.getInt(i) * Math.pow(10, places)) / Math.pow(10, places)); - break; - case INT64: - columnBuilder.writeDouble( - Math.rint(column.getLong(i) * Math.pow(10, places)) / Math.pow(10, places)); - break; - case FLOAT: - columnBuilder.writeDouble( - Math.rint(column.getFloat(i) * Math.pow(10, places)) / Math.pow(10, places)); - break; - case DOUBLE: - columnBuilder.writeDouble( - Math.rint(column.getDouble(i) * Math.pow(10, places)) / Math.pow(10, places)); - break; - case TIMESTAMP: - case STRING: - case BOOLEAN: - case BLOB: - case TEXT: - case DATE: - default: - throw new UnsupportedOperationException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_SOURCE_DATATYPE_ARG_678B759C, - childColumnTransformer.getType().getTypeEnum())); - } + private void transform(Column column, ColumnBuilder columnBuilder, Type sourceType, int i) { + double scale = Math.pow(10, places); + columnBuilder.writeDouble(Math.rint(sourceType.getDouble(column, i) * scale) / scale); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/TryCastFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/TryCastFunctionColumnTransformer.java index ff8a20ddaad3..89676316b11f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/TryCastFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/TryCastFunctionColumnTransformer.java @@ -19,14 +19,12 @@ package org.apache.iotdb.calc.transformation.dag.column.unary.scalar; -import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.read.common.type.TypeEnum; import java.time.ZoneId; @@ -38,47 +36,9 @@ public TryCastFunctionColumnTransformer( } @Override - protected void transform( - Column column, ColumnBuilder columnBuilder, TypeEnum sourceType, Type childType, int i) { + protected void transform(Column column, ColumnBuilder columnBuilder, int i) { try { - switch (sourceType) { - case INT32: - cast(columnBuilder, childType.getInt(column, i)); - break; - case DATE: - castDate(columnBuilder, childType.getInt(column, i)); - break; - case INT64: - cast(columnBuilder, childType.getLong(column, i)); - break; - case TIMESTAMP: - castTimestamp(columnBuilder, childType.getLong(column, i)); - break; - case FLOAT: - cast(columnBuilder, childType.getFloat(column, i)); - break; - case DOUBLE: - cast(columnBuilder, childType.getDouble(column, i)); - break; - case BOOLEAN: - cast(columnBuilder, childType.getBoolean(column, i)); - break; - case TEXT: - case STRING: - castString(columnBuilder, childType.getBinary(column, i)); - break; - case BLOB: - castBlob(columnBuilder, childType.getBinary(column, i)); - break; - case OBJECT: - castObject(columnBuilder, childType.getBinary(column, i)); - break; - default: - throw new UnsupportedOperationException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_SOURCE_DATATYPE_ARG_678B759C, - childColumnTransformer.getType().getTypeEnum())); - } + castSourceValue(column, columnBuilder, i); } catch (IoTDBRuntimeException e) { columnBuilder.appendNull(); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/util/CastFunctionUtils.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/util/CastFunctionUtils.java index 2c0eff17ea26..9a248129fde5 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/util/CastFunctionUtils.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/util/CastFunctionUtils.java @@ -20,20 +20,15 @@ package org.apache.iotdb.calc.transformation.dag.util; import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.queryengine.common.SessionInfo; -import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BytesUtils; -import org.apache.tsfile.utils.DateUtils; import javax.annotation.Nonnull; import java.time.ZoneId; -import java.time.format.DateTimeParseException; public class CastFunctionUtils { public static final String ERROR_MSG = "Unsupported target dataType: %s"; @@ -127,243 +122,11 @@ public static boolean castTextToBoolean(String value) { // used by IrExpressionInterpreter to do constant folding public static Object cast( @Nonnull Object value, Type sourceType, Type targetType, SessionInfo session) { - switch (sourceType.getTypeEnum()) { - case INT32: - int intV = value instanceof Integer ? (int) value : ((Long) value).intValue(); - return castInt(intV, targetType); - case DATE: - int dateV = value instanceof Integer ? (int) value : ((Long) value).intValue(); - return castDate(dateV, targetType, session.getZoneId()); - case INT64: - long longV = (Long) value; - return castLong(longV, targetType); - case TIMESTAMP: - long timestampV = (Long) value; - return castTimestamp(timestampV, targetType, session.getZoneId()); - case FLOAT: - float floatV = value instanceof Float ? (float) value : ((Double) value).floatValue(); - return castFloat(floatV, targetType); - case DOUBLE: - double doubleV = (Double) value; - return castDouble(doubleV, targetType); - case BOOLEAN: - boolean boolV = (Boolean) value; - return castBool(boolV, targetType); - case TEXT: - case STRING: - case BLOB: - Binary binaryV = (Binary) value; - return castBinary(binaryV, targetType, session.getZoneId()); - default: - throw new UnsupportedOperationException( - String.format( - CalcMessages.EXCEPTION_UNSUPPORTED_SOURCE_DATATYPE_ARG_678B759C, - sourceType.getTypeEnum())); - } - } - - private static Object castInt(int value, Type targetType) { - switch (targetType.getTypeEnum()) { - case INT32: - case DATE: - return value; - case INT64: - case TIMESTAMP: - return (long) value; - case FLOAT: - return (float) value; - case DOUBLE: - return (double) value; - case BOOLEAN: - return value != 0; - case TEXT: - case STRING: - return BytesUtils.valueOf(String.valueOf(value)); - case BLOB: - return new Binary(BytesUtils.intToBytes(value)); - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, targetType.getTypeEnum())); - } - } - - private static Object castDate(int value, Type targetType, ZoneId zoneId) { - switch (targetType.getTypeEnum()) { - case INT32: - case DATE: - return value; - case INT64: - return (long) value; - case TIMESTAMP: - return DateTimeUtils.correctPrecision(DateUtils.parseIntToTimestamp(value, zoneId)); - case FLOAT: - return (float) value; - case DOUBLE: - return (double) value; - case BOOLEAN: - return value != 0; - case TEXT: - case STRING: - return BytesUtils.valueOf(DateUtils.formatDate(value)); - case BLOB: - return new Binary(BytesUtils.intToBytes(value)); - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, targetType.getTypeEnum())); - } - } - - private static Object castLong(long value, Type targetType) { - switch (targetType.getTypeEnum()) { - case INT32: - case DATE: - return castLongToInt(value); - case INT64: - case TIMESTAMP: - return value; - case FLOAT: - return (float) value; - case DOUBLE: - return (double) value; - case BOOLEAN: - return value != 0L; - case TEXT: - case STRING: - return BytesUtils.valueOf(String.valueOf(value)); - case BLOB: - return new Binary(BytesUtils.longToBytes(value)); - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, targetType.getTypeEnum())); - } - } - - private static Object castTimestamp(long value, Type targetType, ZoneId zoneId) { - switch (targetType.getTypeEnum()) { - case INT32: - return castLongToInt(value); - case DATE: - return DateUtils.parseDateExpressionToInt(DateTimeUtils.convertToLocalDate(value, zoneId)); - case INT64: - case TIMESTAMP: - return value; - case FLOAT: - return (float) value; - case DOUBLE: - return (double) value; - case BOOLEAN: - return value != 0L; - case TEXT: - case STRING: - return BytesUtils.valueOf(DateTimeUtils.convertLongToDate(value, zoneId)); - case BLOB: - return new Binary(BytesUtils.longToBytes(value)); - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, targetType.getTypeEnum())); - } - } - - private static Object castFloat(float value, Type targetType) { - switch (targetType.getTypeEnum()) { - case INT32: - case DATE: - return castFloatToInt(value); - case INT64: - case TIMESTAMP: - return castFloatToLong(value); - case FLOAT: - return value; - case DOUBLE: - return (double) value; - case BOOLEAN: - return value != 0.0f; - case TEXT: - case STRING: - return BytesUtils.valueOf(String.valueOf(value)); - case BLOB: - return new Binary(BytesUtils.floatToBytes(value)); - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, targetType.getTypeEnum())); - } - } - - private static Object castDouble(double value, Type targetType) { - switch (targetType.getTypeEnum()) { - case INT32: - case DATE: - return castDoubleToInt(value); - case INT64: - case TIMESTAMP: - return castDoubleToLong(value); - case FLOAT: - return castDoubleToFloat(value); - case DOUBLE: - return value; - case BOOLEAN: - return value != 0.0d; - case TEXT: - case STRING: - return BytesUtils.valueOf(String.valueOf(value)); - case BLOB: - return new Binary(BytesUtils.doubleToBytes(value)); - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, targetType.getTypeEnum())); - } - } - - private static Object castBool(boolean value, Type targetType) { - switch (targetType.getTypeEnum()) { - case INT32: - case DATE: - return value ? 1 : 0; - case INT64: - case TIMESTAMP: - return value ? 1L : 0L; - case FLOAT: - return value ? 1.0f : 0.0f; - case DOUBLE: - return value ? 1.0d : 0.0d; - case BOOLEAN: - return value; - case TEXT: - case STRING: - return BytesUtils.valueOf(String.valueOf(value)); - case BLOB: - return new Binary(BytesUtils.boolToBytes(value)); - default: - throw new UnsupportedOperationException(String.format(ERROR_MSG, targetType.getTypeEnum())); - } - } - - private static Object castBinary(Binary value, Type targetType, ZoneId zoneId) { - String stringValue = value.getStringValue(TSFileConfig.STRING_CHARSET); - try { - switch (targetType.getTypeEnum()) { - case INT32: - return Integer.parseInt(stringValue); - case DATE: - return DateUtils.parseDateExpressionToInt(stringValue); - case INT64: - return Long.parseLong(stringValue); - case TIMESTAMP: - return DateTimeUtils.convertDatetimeStrToLong(stringValue, zoneId); - case FLOAT: - return castTextToFloat(stringValue); - case DOUBLE: - return castTextToDouble(stringValue); - case BOOLEAN: - return castTextToBoolean(stringValue); - case TEXT: - case STRING: - case BLOB: - return value; - default: - throw new UnsupportedOperationException( - String.format(ERROR_MSG, targetType.getTypeEnum())); - } - } catch (DateTimeParseException | NumberFormatException e) { - throw new SemanticException( - String.format( - CalcMessages.EXCEPTION_CANNOT_CAST_ARG_ARG_TYPE_8266A2C6, - stringValue, - targetType.getDisplayName())); - } + TypeServices.CastObjectInputService inputService = + TypeServices.CAST_OBJECT_INPUT_SERVICE.call(sourceType); + TypeServices.CastObjectValueService valueService = + TypeServices.CAST_OBJECT_VALUE_SERVICE.call(targetType); + ZoneId zoneId = session == null ? null : session.getZoneId(); + return inputService.cast(value, valueService, zoneId); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/datastructure/row/SerializableRowList.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/datastructure/row/SerializableRowList.java index c2ad397b57e8..277d6a00b8c9 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/datastructure/row/SerializableRowList.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/datastructure/row/SerializableRowList.java @@ -22,11 +22,13 @@ import org.apache.iotdb.calc.exception.QueryProcessException; import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.transformation.datastructure.SerializableList; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.TsBlockSerde; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -36,8 +38,6 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.iotdb.calc.transformation.datastructure.util.BinaryUtils.MIN_ARRAY_HEADER_SIZE; -import static org.apache.iotdb.calc.transformation.datastructure.util.BinaryUtils.MIN_OBJECT_HEADER_SIZE; import static org.apache.iotdb.commons.conf.IoTDBConstant.MB; public class SerializableRowList implements SerializableList { @@ -87,34 +87,10 @@ protected static int calculateCapacity( throws QueryProcessException { int rowLength = ReadWriteIOUtils.LONG_LEN; // timestamp for (TSDataType dataType : dataTypes) { // fields - switch (dataType) { - case INT32: - case DATE: - rowLength += ReadWriteIOUtils.INT_LEN; - break; - case INT64: - case TIMESTAMP: - rowLength += ReadWriteIOUtils.LONG_LEN; - break; - case FLOAT: - rowLength += ReadWriteIOUtils.FLOAT_LEN; - break; - case DOUBLE: - rowLength += ReadWriteIOUtils.DOUBLE_LEN; - break; - case BOOLEAN: - rowLength += ReadWriteIOUtils.BOOLEAN_LEN; - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - rowLength += - MIN_OBJECT_HEADER_SIZE + MIN_ARRAY_HEADER_SIZE + byteArrayLengthForMemoryControl; - break; - default: - throw new UnSupportedDataTypeException(dataType.toString()); - } + rowLength += + TypeServices.MEMORY_USAGE_OF_ONE_SERIALIZABLE_ROW_FIELD_SERVICE + .call(Type.fromTsDataType(dataType)) + .applyAsInt(byteArrayLengthForMemoryControl); } rowLength += ReadWriteIOUtils.BIT_LEN; // null field @@ -195,33 +171,7 @@ private Object[] getRowSkipPrefixNulls(int index) { // Fill value columns for (int i = 0; i < block.length - 1; i++) { - switch (dataTypes[i]) { - case INT32: - case DATE: - row[i] = block[i].getInt(offset); - break; - case INT64: - case TIMESTAMP: - row[i] = block[i].getLong(offset); - break; - case FLOAT: - row[i] = block[i].getFloat(offset); - break; - case DOUBLE: - row[i] = block[i].getDouble(offset); - break; - case BOOLEAN: - row[i] = block[i].getBoolean(offset); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - row[i] = block[i].getBinary(offset); - break; - default: - throw new UnSupportedDataTypeException(dataTypes[i].toString()); - } + row[i] = Type.fromTsDataType(dataTypes[i]).getObject(block[i], offset); } // Fill time column row[block.length - 1] = block[block.length - 1].getLong(offset); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/datastructure/tv/SerializableTVList.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/datastructure/tv/SerializableTVList.java index 70d781c97623..9c3d9f64f7d4 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/datastructure/tv/SerializableTVList.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/datastructure/tv/SerializableTVList.java @@ -21,16 +21,17 @@ import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.transformation.datastructure.SerializableList; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.TsBlockSerde; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.IOException; import java.nio.ByteBuffer; @@ -38,8 +39,6 @@ import java.util.List; import static com.google.common.base.Preconditions.checkState; -import static org.apache.iotdb.calc.transformation.datastructure.util.BinaryUtils.MIN_ARRAY_HEADER_SIZE; -import static org.apache.iotdb.calc.transformation.datastructure.util.BinaryUtils.MIN_OBJECT_HEADER_SIZE; import static org.apache.iotdb.commons.conf.IoTDBConstant.MB; public class SerializableTVList implements SerializableList { @@ -72,36 +71,11 @@ protected SerializableTVList(SerializationRecorder serializationRecorder) { protected static int calculateCapacity(TSDataType dataType, float memoryLimitInMB) { int rowLength = ReadWriteIOUtils.LONG_LEN; // timestamp - switch (dataType) { // value - case INT32: - case DATE: - rowLength += ReadWriteIOUtils.INT_LEN; - break; - case INT64: - case TIMESTAMP: - rowLength += ReadWriteIOUtils.LONG_LEN; - break; - case FLOAT: - rowLength += ReadWriteIOUtils.FLOAT_LEN; - break; - case DOUBLE: - rowLength += ReadWriteIOUtils.DOUBLE_LEN; - break; - case BOOLEAN: - rowLength += ReadWriteIOUtils.BOOLEAN_LEN; - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - rowLength += - MIN_OBJECT_HEADER_SIZE - + MIN_ARRAY_HEADER_SIZE - + SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL; - break; - default: - throw new UnSupportedDataTypeException(dataType.toString()); - } + Type type = Type.fromTsDataType(dataType); + rowLength += + TypeServices.MEMORY_USAGE_OF_ONE_SERIALIZABLE_ROW_FIELD_SERVICE + .call(type) + .applyAsInt(SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL); rowLength += ReadWriteIOUtils.BIT_LEN; int capacity = (int) (memoryLimitInMB * MB / 2 / (rowLength)); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/TypeServices.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/TypeServices.java new file mode 100644 index 000000000000..c31894c848ec --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/TypeServices.java @@ -0,0 +1,3909 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.utils; + +import org.apache.iotdb.calc.execution.operator.process.fill.IFill; +import org.apache.iotdb.calc.execution.operator.process.fill.IFillFilter; +import org.apache.iotdb.calc.execution.operator.process.fill.ILinearFill; +import org.apache.iotdb.calc.execution.operator.process.fill.constant.BinaryConstantFill; +import org.apache.iotdb.calc.execution.operator.process.fill.constant.BooleanConstantFill; +import org.apache.iotdb.calc.execution.operator.process.fill.constant.DoubleConstantFill; +import org.apache.iotdb.calc.execution.operator.process.fill.constant.FloatConstantFill; +import org.apache.iotdb.calc.execution.operator.process.fill.constant.IntConstantFill; +import org.apache.iotdb.calc.execution.operator.process.fill.constant.LongConstantFill; +import org.apache.iotdb.calc.execution.operator.process.fill.identity.IdentityFill; +import org.apache.iotdb.calc.execution.operator.process.fill.identity.IdentityLinearFill; +import org.apache.iotdb.calc.execution.operator.process.fill.linear.DoubleLinearFill; +import org.apache.iotdb.calc.execution.operator.process.fill.linear.FloatLinearFill; +import org.apache.iotdb.calc.execution.operator.process.fill.linear.IntLinearFill; +import org.apache.iotdb.calc.execution.operator.process.fill.linear.LongLinearFill; +import org.apache.iotdb.calc.execution.operator.process.fill.next.BinaryNextFill; +import org.apache.iotdb.calc.execution.operator.process.fill.next.BooleanNextFill; +import org.apache.iotdb.calc.execution.operator.process.fill.next.DoubleNextFill; +import org.apache.iotdb.calc.execution.operator.process.fill.next.FloatNextFill; +import org.apache.iotdb.calc.execution.operator.process.fill.next.IntNextFill; +import org.apache.iotdb.calc.execution.operator.process.fill.next.LongNextFill; +import org.apache.iotdb.calc.execution.operator.process.fill.previous.BinaryPreviousFill; +import org.apache.iotdb.calc.execution.operator.process.fill.previous.BinaryPreviousFillWithTimeDuration; +import org.apache.iotdb.calc.execution.operator.process.fill.previous.BooleanPreviousFill; +import org.apache.iotdb.calc.execution.operator.process.fill.previous.BooleanPreviousFillWithTimeDuration; +import org.apache.iotdb.calc.execution.operator.process.fill.previous.DoublePreviousFill; +import org.apache.iotdb.calc.execution.operator.process.fill.previous.DoublePreviousFillWithTimeDuration; +import org.apache.iotdb.calc.execution.operator.process.fill.previous.FloatPreviousFill; +import org.apache.iotdb.calc.execution.operator.process.fill.previous.FloatPreviousFillWithTimeDuration; +import org.apache.iotdb.calc.execution.operator.process.fill.previous.IntPreviousFill; +import org.apache.iotdb.calc.execution.operator.process.fill.previous.IntPreviousFillWithTimeDuration; +import org.apache.iotdb.calc.execution.operator.process.fill.previous.LongPreviousFill; +import org.apache.iotdb.calc.execution.operator.process.fill.previous.LongPreviousFillWithTimeDuration; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.AscBinaryTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.AscBooleanTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.AscDoubleTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.AscFloatTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.AscIntTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.AscLongTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.DescBinaryTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.DescBooleanTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.DescDoubleTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.DescFloatTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.DescIntTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.DescLongTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.JoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.window.partition.Partition; +import org.apache.iotdb.calc.execution.operator.process.window.utils.ColumnList; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AbstractApproxPercentileAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.BinaryApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.BlobApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.BooleanApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.DoubleApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.FloatApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.IntApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.LongApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.HyperLogLog; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.AbstractGroupedApproxPercentileAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.BinaryGroupedApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.BlobGroupedApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.BooleanGroupedApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.DoubleGroupedApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.FloatGroupedApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedMaxMinByBaseAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.IntGroupedApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.LongGroupedApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BinaryBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BooleanBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.DoubleBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.FloatBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.plan.relational.planner.CastToBlobLiteralVisitor; +import org.apache.iotdb.calc.plan.relational.planner.CastToBooleanLiteralVisitor; +import org.apache.iotdb.calc.plan.relational.planner.CastToDateLiteralVisitor; +import org.apache.iotdb.calc.plan.relational.planner.CastToDoubleLiteralVisitor; +import org.apache.iotdb.calc.plan.relational.planner.CastToFloatLiteralVisitor; +import org.apache.iotdb.calc.plan.relational.planner.CastToInt32LiteralVisitor; +import org.apache.iotdb.calc.plan.relational.planner.CastToInt64LiteralVisitor; +import org.apache.iotdb.calc.plan.relational.planner.CastToStringLiteralVisitor; +import org.apache.iotdb.calc.plan.relational.planner.CastToTimestampLiteralVisitor; +import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.BinaryGreatestColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.BinaryLeastColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.BooleanGreatestColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.BooleanLeastColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.DoubleGreatestColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.DoubleLeastColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.FloatGreatestColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.FloatLeastColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.InBinaryMultiColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.InBooleanMultiColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.InDoubleMultiColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.InFloatMultiColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.InInt32MultiColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.InInt64MultiColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.InMultiColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.Int32GreatestColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.Int32LeastColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.Int64GreatestColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.Int64LeastColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.util.CastFunctionUtils; +import org.apache.iotdb.calc.utils.datastructure.SortKey; +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BinaryLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FloatLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.GenericLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Literal; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +import java.time.ZoneId; +import java.time.format.DateTimeParseException; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.function.IntConsumer; +import java.util.function.IntFunction; +import java.util.function.IntUnaryOperator; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.apache.iotdb.calc.transformation.datastructure.util.BinaryUtils.MIN_ARRAY_HEADER_SIZE; +import static org.apache.iotdb.calc.transformation.datastructure.util.BinaryUtils.MIN_OBJECT_HEADER_SIZE; + +public class TypeServices { + + private static final IdentityLinearFill IDENTITY_LINEAR_FILL = new IdentityLinearFill(); + private static final IdentityFill IDENTITY_FILL = new IdentityFill(); + + // Unsupported fill types defer their exception until the returned strategy is used so check() + // can still validate that every TypeEnum has a service entry. + public static final TypeService> LINEAR_FILL_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> IntLinearFill::new; + case INT64, TIMESTAMP -> LongLinearFill::new; + case FLOAT -> FloatLinearFill::new; + case DOUBLE -> DoubleLinearFill::new; + case BOOLEAN, TEXT, STRING, BLOB, OBJECT -> () -> IDENTITY_LINEAR_FILL; + case ROW, UNKNOWN, VECTOR -> + () -> { + throw unsupportedDataType(type); + }; + }; + + public static final TypeService> PREVIOUS_FILL_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + filter -> + filter == null + ? new BooleanPreviousFill() + : new BooleanPreviousFillWithTimeDuration(filter); + case TEXT, STRING, BLOB, OBJECT -> + filter -> + filter == null + ? new BinaryPreviousFill() + : new BinaryPreviousFillWithTimeDuration(filter); + case INT32, DATE -> + filter -> + filter == null + ? new IntPreviousFill() + : new IntPreviousFillWithTimeDuration(filter); + case INT64, TIMESTAMP -> + filter -> + filter == null + ? new LongPreviousFill() + : new LongPreviousFillWithTimeDuration(filter); + case FLOAT -> + filter -> + filter == null + ? new FloatPreviousFill() + : new FloatPreviousFillWithTimeDuration(filter); + case DOUBLE -> + filter -> + filter == null + ? new DoublePreviousFill() + : new DoublePreviousFillWithTimeDuration(filter); + case ROW, UNKNOWN, VECTOR -> + filter -> { + throw unsupportedDataType(type); + }; + }; + + public static final TypeService> NEXT_FILL_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> BooleanNextFill::new; + case TEXT, STRING, BLOB, OBJECT -> BinaryNextFill::new; + case INT32, DATE -> IntNextFill::new; + case INT64, TIMESTAMP -> LongNextFill::new; + case FLOAT -> FloatNextFill::new; + case DOUBLE -> DoubleNextFill::new; + case ROW, UNKNOWN, VECTOR -> + filter -> { + throw unsupportedDataType(type); + }; + }; + + public static final TypeService> JOIN_KEY_COMPARATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + ascending -> + ascending + ? AscIntTypeJoinKeyComparator.getInstance() + : DescIntTypeJoinKeyComparator.getInstance(); + case INT64, TIMESTAMP -> + ascending -> + ascending + ? AscLongTypeJoinKeyComparator.getInstance() + : DescLongTypeJoinKeyComparator.getInstance(); + case FLOAT -> + ascending -> + ascending + ? AscFloatTypeJoinKeyComparator.getInstance() + : DescFloatTypeJoinKeyComparator.getInstance(); + case DOUBLE -> + ascending -> + ascending + ? AscDoubleTypeJoinKeyComparator.getInstance() + : DescDoubleTypeJoinKeyComparator.getInstance(); + case BOOLEAN -> + ascending -> + ascending + ? AscBooleanTypeJoinKeyComparator.getInstance() + : DescBooleanTypeJoinKeyComparator.getInstance(); + case STRING, BLOB, TEXT -> + ascending -> + ascending + ? AscBinaryTypeJoinKeyComparator.getInstance() + : DescBinaryTypeJoinKeyComparator.getInstance(); + case OBJECT, ROW, UNKNOWN, VECTOR -> + ascending -> { + throw new UnsupportedOperationException( + CalcMessages.UNSUPPORTED_DATA_TYPE + type); + }; + }; + + public static final TypeService>> MERGE_SORT_COMPARATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + index -> + Comparator.comparingInt( + sortKey -> type.getInt(sortKey.tsBlock.getColumn(index), sortKey.rowIndex)); + case INT64, TIMESTAMP -> + index -> + Comparator.comparingLong( + sortKey -> + type.getLong(sortKey.tsBlock.getColumn(index), sortKey.rowIndex)); + case FLOAT -> + index -> + Comparator.comparingDouble( + sortKey -> + type.getFloat(sortKey.tsBlock.getColumn(index), sortKey.rowIndex)); + case DOUBLE -> + index -> + Comparator.comparingDouble( + sortKey -> + type.getDouble(sortKey.tsBlock.getColumn(index), sortKey.rowIndex)); + case TEXT, STRING, BLOB, OBJECT -> + index -> + Comparator.comparing( + sortKey -> + type.getBinary(sortKey.tsBlock.getColumn(index), sortKey.rowIndex)); + case BOOLEAN -> + index -> + Comparator.comparing( + sortKey -> + type.getBoolean(sortKey.tsBlock.getColumn(index), sortKey.rowIndex)); + // TypeService.check() must be able to build a strategy for every enum value. + case ROW, UNKNOWN, VECTOR -> + index -> { + throw new IllegalArgumentException( + String.format(CalcMessages.DATA_TYPE_CANNOT_BE_ORDERED, type)); + }; + }; + + public static final TypeService MEMORY_USAGE_OF_ONE_MERGE_SORT_KEY_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> 1; + case INT32, FLOAT, DATE -> 4; + case INT64, DOUBLE, TIMESTAMP -> 8; + case TEXT, STRING, BLOB, OBJECT -> 16; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(CalcMessages.UNKNOWN_DATATYPE + type) + .setChecked(true); + }; + + public static final TypeService + MEMORY_USAGE_OF_ONE_SERIALIZABLE_ROW_FIELD_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> ignored -> ReadWriteIOUtils.INT_LEN; + case INT64, TIMESTAMP -> ignored -> ReadWriteIOUtils.LONG_LEN; + case FLOAT -> ignored -> ReadWriteIOUtils.FLOAT_LEN; + case DOUBLE -> ignored -> ReadWriteIOUtils.DOUBLE_LEN; + case BOOLEAN -> ignored -> ReadWriteIOUtils.BOOLEAN_LEN; + case TEXT, BLOB, STRING, OBJECT -> + byteArrayLength -> + MIN_OBJECT_HEADER_SIZE + MIN_ARRAY_HEADER_SIZE + byteArrayLength; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(type.toString()).setChecked(true); + }; + + public static final TypeService VALUE_FILL_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (literal, zoneId) -> { + Boolean value = literal.accept(new CastToBooleanLiteralVisitor(), null); + return value == null ? IDENTITY_FILL : new BooleanConstantFill(value); + }; + case TEXT, STRING -> + (literal, zoneId) -> { + Binary value = + literal.accept( + new CastToStringLiteralVisitor(TSFileConfig.STRING_CHARSET), null); + return value == null ? IDENTITY_FILL : new BinaryConstantFill(value); + }; + case BLOB -> + (literal, zoneId) -> { + Binary value = literal.accept(new CastToBlobLiteralVisitor(), null); + return value == null ? IDENTITY_FILL : new BinaryConstantFill(value); + }; + case INT32 -> + (literal, zoneId) -> { + Integer value = literal.accept(new CastToInt32LiteralVisitor(), null); + return value == null ? IDENTITY_FILL : new IntConstantFill(value); + }; + case DATE -> + (literal, zoneId) -> { + Integer value = literal.accept(new CastToDateLiteralVisitor(), null); + return value == null ? IDENTITY_FILL : new IntConstantFill(value); + }; + case INT64 -> + (literal, zoneId) -> { + Long value = literal.accept(new CastToInt64LiteralVisitor(), null); + return value == null ? IDENTITY_FILL : new LongConstantFill(value); + }; + case TIMESTAMP -> + (literal, zoneId) -> { + Long value = literal.accept(new CastToTimestampLiteralVisitor(zoneId), null); + return value == null ? IDENTITY_FILL : new LongConstantFill(value); + }; + case FLOAT -> + (literal, zoneId) -> { + Float value = literal.accept(new CastToFloatLiteralVisitor(), null); + return value == null ? IDENTITY_FILL : new FloatConstantFill(value); + }; + case DOUBLE -> + (literal, zoneId) -> { + Double value = literal.accept(new CastToDoubleLiteralVisitor(), null); + return value == null ? IDENTITY_FILL : new DoubleConstantFill(value); + }; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (literal, zoneId) -> { + throw unsupportedDataType(type); + }; + }; + + /** Reads values for a cast using the source type's representation. */ + public static final TypeService CAST_INPUT_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (column, columnBuilder, position, castValueService, zoneId) -> + castValueService.castInt(columnBuilder, type.getInt(column, position)); + case DATE -> + (column, columnBuilder, position, castValueService, zoneId) -> + castValueService.castDate(columnBuilder, type.getInt(column, position), zoneId); + case INT64 -> + (column, columnBuilder, position, castValueService, zoneId) -> + castValueService.castLong(columnBuilder, type.getLong(column, position)); + case TIMESTAMP -> + (column, columnBuilder, position, castValueService, zoneId) -> + castValueService.castTimestamp( + columnBuilder, type.getLong(column, position), zoneId); + case FLOAT -> + (column, columnBuilder, position, castValueService, zoneId) -> + castValueService.castFloat(columnBuilder, type.getFloat(column, position)); + case DOUBLE -> + (column, columnBuilder, position, castValueService, zoneId) -> + castValueService.castDouble(columnBuilder, type.getDouble(column, position)); + case BOOLEAN -> + (column, columnBuilder, position, castValueService, zoneId) -> + castValueService.castBoolean(columnBuilder, type.getBoolean(column, position)); + case TEXT, STRING -> + (column, columnBuilder, position, castValueService, zoneId) -> + castValueService.castString( + columnBuilder, type.getBinary(column, position), zoneId); + case BLOB -> + (column, columnBuilder, position, castValueService, zoneId) -> + castValueService.castBlob( + columnBuilder, type.getBinary(column, position), zoneId); + case OBJECT -> + (column, columnBuilder, position, castValueService, zoneId) -> + castValueService.castObject(columnBuilder, type.getBinary(column, position)); + case ROW, UNKNOWN, VECTOR -> + (column, columnBuilder, position, castValueService, zoneId) -> { + throw unsupportedCastSource(type); + }; + }; + + /** Writes values produced by a cast into a column using the target type's representation. */ + public static final TypeService CAST_VALUE_SERVICE = + type -> + new CastValueService() { + @Override + public void castInt(ColumnBuilder columnBuilder, int value) { + switch (type.getTypeEnum()) { + case INT32, DATE -> type.writeInt(columnBuilder, value); + case INT64, TIMESTAMP -> type.writeLong(columnBuilder, value); + case FLOAT -> type.writeFloat(columnBuilder, value); + case DOUBLE -> type.writeDouble(columnBuilder, value); + case BOOLEAN -> type.writeBoolean(columnBuilder, value != 0); + case TEXT, STRING -> + type.writeBinary(columnBuilder, BytesUtils.valueOf(String.valueOf(value))); + case BLOB -> + type.writeBinary(columnBuilder, new Binary(BytesUtils.intToBytes(value))); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + } + } + + @Override + public void castDate(ColumnBuilder columnBuilder, int value, ZoneId zoneId) { + switch (type.getTypeEnum()) { + case INT32, DATE -> type.writeInt(columnBuilder, value); + case INT64 -> type.writeLong(columnBuilder, value); + case TIMESTAMP -> + type.writeLong( + columnBuilder, + DateTimeUtils.correctPrecision( + DateUtils.parseIntToTimestamp(value, zoneId))); + case FLOAT -> type.writeFloat(columnBuilder, value); + case DOUBLE -> type.writeDouble(columnBuilder, value); + case BOOLEAN -> type.writeBoolean(columnBuilder, value != 0); + case TEXT, STRING -> + type.writeBinary( + columnBuilder, BytesUtils.valueOf(DateUtils.formatDate(value))); + case BLOB -> + type.writeBinary(columnBuilder, new Binary(BytesUtils.intToBytes(value))); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + } + } + + @Override + public void castTimestamp(ColumnBuilder columnBuilder, long value, ZoneId zoneId) { + try { + switch (type.getTypeEnum()) { + case INT32 -> + type.writeInt(columnBuilder, CastFunctionUtils.castLongToInt(value)); + case DATE -> + type.writeInt( + columnBuilder, + DateUtils.parseDateExpressionToInt( + DateTimeUtils.convertToLocalDate(value, zoneId))); + case INT64, TIMESTAMP -> type.writeLong(columnBuilder, value); + case FLOAT -> type.writeFloat(columnBuilder, value); + case DOUBLE -> type.writeDouble(columnBuilder, value); + case BOOLEAN -> type.writeBoolean(columnBuilder, value != 0L); + case TEXT, STRING -> + type.writeBinary( + columnBuilder, + BytesUtils.valueOf(DateTimeUtils.convertLongToDate(value, zoneId))); + case BLOB -> + type.writeBinary(columnBuilder, new Binary(BytesUtils.longToBytes(value))); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + } + } catch (DateTimeParseException e) { + throw new IoTDBRuntimeException( + "Year must be between 1000 and 9999.", + org.apache.iotdb.rpc.TSStatusCode.DATE_OUT_OF_RANGE.getStatusCode(), + true); + } + } + + @Override + public void castLong(ColumnBuilder columnBuilder, long value) { + switch (type.getTypeEnum()) { + case INT32, DATE -> + type.writeInt(columnBuilder, CastFunctionUtils.castLongToInt(value)); + case INT64, TIMESTAMP -> type.writeLong(columnBuilder, value); + case FLOAT -> type.writeFloat(columnBuilder, value); + case DOUBLE -> type.writeDouble(columnBuilder, value); + case BOOLEAN -> type.writeBoolean(columnBuilder, value != 0L); + case TEXT, STRING -> + type.writeBinary(columnBuilder, BytesUtils.valueOf(String.valueOf(value))); + case BLOB -> + type.writeBinary(columnBuilder, new Binary(BytesUtils.longToBytes(value))); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + } + } + + @Override + public void castFloat(ColumnBuilder columnBuilder, float value) { + switch (type.getTypeEnum()) { + case INT32, DATE -> + type.writeInt(columnBuilder, CastFunctionUtils.castFloatToInt(value)); + case INT64, TIMESTAMP -> + type.writeLong(columnBuilder, CastFunctionUtils.castFloatToLong(value)); + case FLOAT -> type.writeFloat(columnBuilder, value); + case DOUBLE -> type.writeDouble(columnBuilder, value); + case BOOLEAN -> type.writeBoolean(columnBuilder, value != 0.0f); + case TEXT, STRING -> + type.writeBinary(columnBuilder, BytesUtils.valueOf(String.valueOf(value))); + case BLOB -> + type.writeBinary(columnBuilder, new Binary(BytesUtils.floatToBytes(value))); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + } + } + + @Override + public void castDouble(ColumnBuilder columnBuilder, double value) { + switch (type.getTypeEnum()) { + case INT32, DATE -> + type.writeInt(columnBuilder, CastFunctionUtils.castDoubleToInt(value)); + case INT64, TIMESTAMP -> + type.writeLong(columnBuilder, CastFunctionUtils.castDoubleToLong(value)); + case FLOAT -> + type.writeFloat(columnBuilder, CastFunctionUtils.castDoubleToFloat(value)); + case DOUBLE -> type.writeDouble(columnBuilder, value); + case BOOLEAN -> type.writeBoolean(columnBuilder, value != 0.0); + case TEXT, STRING -> + type.writeBinary(columnBuilder, BytesUtils.valueOf(String.valueOf(value))); + case BLOB -> + type.writeBinary(columnBuilder, new Binary(BytesUtils.doubleToBytes(value))); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + } + } + + @Override + public void castBoolean(ColumnBuilder columnBuilder, boolean value) { + switch (type.getTypeEnum()) { + case INT32, DATE -> type.writeInt(columnBuilder, value ? 1 : 0); + case INT64, TIMESTAMP -> type.writeLong(columnBuilder, value ? 1L : 0L); + case FLOAT -> type.writeFloat(columnBuilder, value ? 1.0f : 0); + case DOUBLE -> type.writeDouble(columnBuilder, value ? 1.0 : 0); + case BOOLEAN -> type.writeBoolean(columnBuilder, value); + case TEXT, STRING -> + type.writeBinary(columnBuilder, BytesUtils.valueOf(String.valueOf(value))); + case BLOB -> + type.writeBinary(columnBuilder, new Binary(BytesUtils.boolToBytes(value))); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + } + } + + @Override + public void castString(ColumnBuilder columnBuilder, Binary value, ZoneId zoneId) { + String stringValue = value.getStringValue(TSFileConfig.STRING_CHARSET); + try { + switch (type.getTypeEnum()) { + case INT32 -> type.writeInt(columnBuilder, Integer.parseInt(stringValue)); + case DATE -> + type.writeInt(columnBuilder, DateUtils.parseDateExpressionToInt(stringValue)); + case INT64 -> type.writeLong(columnBuilder, Long.parseLong(stringValue)); + case TIMESTAMP -> + type.writeLong( + columnBuilder, + DateTimeUtils.convertDatetimeStrToLong(stringValue, zoneId)); + case FLOAT -> + type.writeFloat( + columnBuilder, CastFunctionUtils.castTextToFloat(stringValue)); + case DOUBLE -> + type.writeDouble( + columnBuilder, CastFunctionUtils.castTextToDouble(stringValue)); + case BOOLEAN -> + type.writeBoolean( + columnBuilder, CastFunctionUtils.castTextToBoolean(stringValue)); + case TEXT, STRING, BLOB, OBJECT -> type.writeBinary(columnBuilder, value); + case ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + } + } catch (DateTimeParseException | NumberFormatException e) { + throw new SemanticException( + String.format("Cannot cast %s to %s type", stringValue, type.getDisplayName())); + } + } + + @Override + public void castBlob(ColumnBuilder columnBuilder, Binary value, ZoneId zoneId) { + String stringValue = BytesUtils.parseBlobByteArrayToString(value.getValues()); + try { + switch (type.getTypeEnum()) { + case INT32 -> type.writeInt(columnBuilder, Integer.parseInt(stringValue)); + case DATE -> + type.writeInt(columnBuilder, DateUtils.parseDateExpressionToInt(stringValue)); + case INT64 -> type.writeLong(columnBuilder, Long.parseLong(stringValue)); + case TIMESTAMP -> + type.writeLong( + columnBuilder, + DateTimeUtils.convertDatetimeStrToLong(stringValue, zoneId)); + case FLOAT -> + type.writeFloat( + columnBuilder, CastFunctionUtils.castTextToFloat(stringValue)); + case DOUBLE -> + type.writeDouble( + columnBuilder, CastFunctionUtils.castTextToDouble(stringValue)); + case BOOLEAN -> + type.writeBoolean( + columnBuilder, CastFunctionUtils.castTextToBoolean(stringValue)); + case TEXT, STRING, BLOB -> type.writeBinary(columnBuilder, value); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + } + } catch (DateTimeParseException | NumberFormatException e) { + throw new SemanticException( + String.format("Cannot cast %s to %s type", stringValue, type.getDisplayName())); + } + } + + @Override + public void castObject(ColumnBuilder columnBuilder, Binary value) { + String stringValue = BytesUtils.parseObjectByteArrayToString(value.getValues()); + switch (type.getTypeEnum()) { + case STRING -> + type.writeBinary( + columnBuilder, BytesUtils.valueOf(String.valueOf(stringValue))); + case INT32, + INT64, + FLOAT, + DOUBLE, + BOOLEAN, + DATE, + TIMESTAMP, + TEXT, + BLOB, + OBJECT, + ROW, + UNKNOWN, + VECTOR -> + throw unsupportedCastTarget(type); + } + } + }; + + public static final TypeService CAST_OBJECT_VALUE_SERVICE = + type -> + new CastObjectValueService() { + @Override + public Object castInt(int value) { + return switch (type.getTypeEnum()) { + case INT32, DATE -> value; + case INT64, TIMESTAMP -> (long) value; + case FLOAT -> (float) value; + case DOUBLE -> (double) value; + case BOOLEAN -> value != 0; + case TEXT, STRING -> BytesUtils.valueOf(String.valueOf(value)); + case BLOB -> new Binary(BytesUtils.intToBytes(value)); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + }; + } + + @Override + public Object castDate(int value, ZoneId zoneId) { + return switch (type.getTypeEnum()) { + case INT32, DATE -> value; + case INT64 -> (long) value; + case TIMESTAMP -> + DateTimeUtils.correctPrecision(DateUtils.parseIntToTimestamp(value, zoneId)); + case FLOAT -> (float) value; + case DOUBLE -> (double) value; + case BOOLEAN -> value != 0; + case TEXT, STRING -> BytesUtils.valueOf(DateUtils.formatDate(value)); + case BLOB -> new Binary(BytesUtils.intToBytes(value)); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + }; + } + + @Override + public Object castTimestamp(long value, ZoneId zoneId) { + return switch (type.getTypeEnum()) { + case INT32 -> CastFunctionUtils.castLongToInt(value); + case DATE -> + DateUtils.parseDateExpressionToInt( + DateTimeUtils.convertToLocalDate(value, zoneId)); + case INT64, TIMESTAMP -> value; + case FLOAT -> (float) value; + case DOUBLE -> (double) value; + case BOOLEAN -> value != 0L; + case TEXT, STRING -> + BytesUtils.valueOf(DateTimeUtils.convertLongToDate(value, zoneId)); + case BLOB -> new Binary(BytesUtils.longToBytes(value)); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + }; + } + + @Override + public Object castLong(long value) { + return switch (type.getTypeEnum()) { + case INT32, DATE -> CastFunctionUtils.castLongToInt(value); + case INT64, TIMESTAMP -> value; + case FLOAT -> (float) value; + case DOUBLE -> (double) value; + case BOOLEAN -> value != 0L; + case TEXT, STRING -> BytesUtils.valueOf(String.valueOf(value)); + case BLOB -> new Binary(BytesUtils.longToBytes(value)); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + }; + } + + @Override + public Object castFloat(float value) { + return switch (type.getTypeEnum()) { + case INT32, DATE -> CastFunctionUtils.castFloatToInt(value); + case INT64, TIMESTAMP -> CastFunctionUtils.castFloatToLong(value); + case FLOAT -> value; + case DOUBLE -> (double) value; + case BOOLEAN -> value != 0.0f; + case TEXT, STRING -> BytesUtils.valueOf(String.valueOf(value)); + case BLOB -> new Binary(BytesUtils.floatToBytes(value)); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + }; + } + + @Override + public Object castDouble(double value) { + return switch (type.getTypeEnum()) { + case INT32, DATE -> CastFunctionUtils.castDoubleToInt(value); + case INT64, TIMESTAMP -> CastFunctionUtils.castDoubleToLong(value); + case FLOAT -> CastFunctionUtils.castDoubleToFloat(value); + case DOUBLE -> value; + case BOOLEAN -> value != 0.0d; + case TEXT, STRING -> BytesUtils.valueOf(String.valueOf(value)); + case BLOB -> new Binary(BytesUtils.doubleToBytes(value)); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + }; + } + + @Override + public Object castBoolean(boolean value) { + return switch (type.getTypeEnum()) { + case INT32, DATE -> value ? 1 : 0; + case INT64, TIMESTAMP -> value ? 1L : 0L; + case FLOAT -> value ? 1.0f : 0.0f; + case DOUBLE -> value ? 1.0d : 0.0d; + case BOOLEAN -> value; + case TEXT, STRING -> BytesUtils.valueOf(String.valueOf(value)); + case BLOB -> new Binary(BytesUtils.boolToBytes(value)); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + }; + } + + @Override + public Object castBinary(Binary value, ZoneId zoneId) { + String stringValue = value.getStringValue(TSFileConfig.STRING_CHARSET); + try { + return switch (type.getTypeEnum()) { + case INT32 -> Integer.parseInt(stringValue); + case DATE -> DateUtils.parseDateExpressionToInt(stringValue); + case INT64 -> Long.parseLong(stringValue); + case TIMESTAMP -> DateTimeUtils.convertDatetimeStrToLong(stringValue, zoneId); + case FLOAT -> CastFunctionUtils.castTextToFloat(stringValue); + case DOUBLE -> CastFunctionUtils.castTextToDouble(stringValue); + case BOOLEAN -> CastFunctionUtils.castTextToBoolean(stringValue); + case TEXT, STRING, BLOB -> value; + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedCastTarget(type); + }; + } catch (DateTimeParseException | NumberFormatException e) { + throw new SemanticException( + String.format("Cannot cast %s to %s type", stringValue, type.getDisplayName())); + } + } + }; + + public static final TypeService CAST_OBJECT_INPUT_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (value, castValueService, zoneId) -> + castValueService.castInt( + value instanceof Integer ? (int) value : ((Long) value).intValue()); + case DATE -> + (value, castValueService, zoneId) -> + castValueService.castDate( + value instanceof Integer ? (int) value : ((Long) value).intValue(), zoneId); + case INT64 -> + (value, castValueService, zoneId) -> castValueService.castLong((Long) value); + case TIMESTAMP -> + (value, castValueService, zoneId) -> + castValueService.castTimestamp((Long) value, zoneId); + case FLOAT -> + (value, castValueService, zoneId) -> + castValueService.castFloat( + value instanceof Float ? (float) value : ((Double) value).floatValue()); + case DOUBLE -> + (value, castValueService, zoneId) -> castValueService.castDouble((Double) value); + case BOOLEAN -> + (value, castValueService, zoneId) -> castValueService.castBoolean((Boolean) value); + case TEXT, STRING, BLOB -> + (value, castValueService, zoneId) -> + castValueService.castBinary((Binary) value, zoneId); + case OBJECT, ROW, UNKNOWN, VECTOR -> + (value, castValueService, zoneId) -> { + throw unsupportedCastSource(type); + }; + }; + + public static final TypeService FORMAT_VALUE_CONVERTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case UNKNOWN -> (column, position, zoneId) -> null; + case INT32, INT64, FLOAT, DOUBLE, BOOLEAN, TEXT, STRING, BLOB -> + (column, position, zoneId) -> column.getObject(position); + // Java Formatter expects temporal objects for DATE and TIMESTAMP arguments. + case DATE -> + (column, position, zoneId) -> + DateUtils.parseIntToLocalDate(column.getInt(position)); + case TIMESTAMP -> + (column, position, zoneId) -> + DateTimeUtils.convertToZonedDateTime(column.getLong(position), zoneId); + case OBJECT, ROW, VECTOR -> + (column, position, zoneId) -> { + throw new UnsupportedOperationException( + CalcMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()); + }; + }; + + public static final TypeService + GROUPED_APPROX_MOST_FREQUENT_ACCUMULATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> new BooleanGroupedApproxMostFrequentAccumulator(); + case INT32, DATE -> new IntGroupedApproxMostFrequentAccumulator(); + case INT64, TIMESTAMP -> new LongGroupedApproxMostFrequentAccumulator(); + case FLOAT -> new FloatGroupedApproxMostFrequentAccumulator(); + case DOUBLE -> new DoubleGroupedApproxMostFrequentAccumulator(); + case TEXT, STRING -> new BinaryGroupedApproxMostFrequentAccumulator(); + case BLOB -> new BlobGroupedApproxMostFrequentAccumulator(); + case OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupportedApproxMostFrequentDataType(type); + }; + + public static final TypeService APPROX_MOST_FREQUENT_ACCUMULATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> new BooleanApproxMostFrequentAccumulator(); + case INT32, DATE -> new IntApproxMostFrequentAccumulator(); + case INT64, TIMESTAMP -> new LongApproxMostFrequentAccumulator(); + case FLOAT -> new FloatApproxMostFrequentAccumulator(); + case DOUBLE -> new DoubleApproxMostFrequentAccumulator(); + case TEXT, STRING -> new BinaryApproxMostFrequentAccumulator(); + case BLOB -> new BlobApproxMostFrequentAccumulator(); + case OBJECT, ROW, UNKNOWN, VECTOR -> throw unsupportedApproxMostFrequentDataType(type); + }; + + public static final TypeService, ColumnTransformer>> + GREATEST_COLUMN_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + columnTransformers -> + new BooleanGreatestColumnTransformer(type, columnTransformers); + case INT32, DATE -> + columnTransformers -> + new Int32GreatestColumnTransformer(type, columnTransformers); + case INT64, TIMESTAMP -> + columnTransformers -> + new Int64GreatestColumnTransformer(type, columnTransformers); + case FLOAT -> + columnTransformers -> + new FloatGreatestColumnTransformer(type, columnTransformers); + case DOUBLE -> + columnTransformers -> + new DoubleGreatestColumnTransformer(type, columnTransformers); + case STRING, TEXT -> + columnTransformers -> + new BinaryGreatestColumnTransformer(type, columnTransformers); + case BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + columnTransformers -> { + throw unsupportedGreatestLeastDataType(type); + }; + }; + + public static final TypeService, ColumnTransformer>> + LEAST_COLUMN_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + columnTransformers -> + new BooleanLeastColumnTransformer(type, columnTransformers); + case INT32, DATE -> + columnTransformers -> new Int32LeastColumnTransformer(type, columnTransformers); + case INT64, TIMESTAMP -> + columnTransformers -> new Int64LeastColumnTransformer(type, columnTransformers); + case FLOAT -> + columnTransformers -> new FloatLeastColumnTransformer(type, columnTransformers); + case DOUBLE -> + columnTransformers -> + new DoubleLeastColumnTransformer(type, columnTransformers); + case STRING, TEXT -> + columnTransformers -> + new BinaryLeastColumnTransformer(type, columnTransformers); + case BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + columnTransformers -> { + throw unsupportedGreatestLeastDataType(type); + }; + }; + + /** + * Creates a single-row column containing the value at the requested row. Delegating the write to + * {@link Type} preserves type-specific column metadata (for example, DATE) without another + * TsDataType dispatch. + */ + public static final TypeService UPDATE_LAST_ROW_SERVICE = + type -> + (column, rowIndex) -> { + // Use the status overload so specialized types such as DATE retain their data type. + ColumnBuilder columnBuilder = type.createColumnBuilder(null, 1); + type.write(columnBuilder, column, rowIndex); + return columnBuilder.build(); + }; + + /** Copies a column value into a reusable primitive result without boxing. */ + public static final TypeService + PRIMITIVE_COLUMN_VALUE_SETTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (target, column, position) -> target.setInt(type.getInt(column, position)); + case INT64, TIMESTAMP -> + (target, column, position) -> target.setLong(type.getLong(column, position)); + case FLOAT -> + (target, column, position) -> target.setFloat(type.getFloat(column, position)); + case DOUBLE -> + (target, column, position) -> + target.setDouble(type.getDouble(column, position)); + case BOOLEAN -> + (target, column, position) -> + target.setBoolean(type.getBoolean(column, position)); + case TEXT, STRING, BLOB, OBJECT -> + (target, column, position) -> + target.setBinary(type.getBinary(column, position)); + case ROW, UNKNOWN, VECTOR -> + (target, column, position) -> { + throw new UnSupportedDataTypeException( + CalcMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()) + .setChecked(true); + }; + }; + + /** Writes a statistics value into a reusable primitive result. */ + public static final TypeService STATISTICS_VALUE_SETTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (target, value, exceptionSupplier) -> target.setInt(((Number) value).intValue()); + case INT64, TIMESTAMP -> + (target, value, exceptionSupplier) -> target.setLong(((Number) value).longValue()); + case FLOAT -> + (target, value, exceptionSupplier) -> + target.setFloat(((Number) value).floatValue()); + case DOUBLE -> + (target, value, exceptionSupplier) -> + target.setDouble(((Number) value).doubleValue()); + case BOOLEAN -> + (target, value, exceptionSupplier) -> target.setBoolean((boolean) value); + case TEXT, STRING, BLOB, OBJECT -> + (target, value, exceptionSupplier) -> target.setBinary(toBinary(value)); + case ROW, UNKNOWN, VECTOR -> + (target, value, exceptionSupplier) -> { + throw exceptionSupplier.get(); + }; + }; + + public static final TypeService IN_COLUMN_VALUE_MATCHER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + values -> { + Set intSet = new HashSet<>(); + for (String value : values) { + try { + intSet.add(Integer.valueOf(value)); + } catch (IllegalArgumentException e) { + throw new SemanticException( + String.format( + CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum())); + } + } + return (column, position) -> intSet.contains(type.getInt(column, position)); + }; + case INT64, TIMESTAMP -> + values -> { + Set longSet = new HashSet<>(); + for (String value : values) { + try { + longSet.add(Long.valueOf(value)); + } catch (IllegalArgumentException e) { + throw new SemanticException( + String.format( + CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum())); + } + } + return (column, position) -> longSet.contains(type.getLong(column, position)); + }; + case FLOAT -> + values -> { + Set floatSet = new HashSet<>(); + for (String value : values) { + try { + floatSet.add(Float.valueOf(value)); + } catch (IllegalArgumentException e) { + throw new SemanticException( + String.format( + CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum())); + } + } + return (column, position) -> floatSet.contains(type.getFloat(column, position)); + }; + case DOUBLE -> + values -> { + Set doubleSet = new HashSet<>(); + for (String value : values) { + try { + doubleSet.add(Double.valueOf(value)); + } catch (IllegalArgumentException e) { + throw new SemanticException( + String.format( + CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum())); + } + } + return (column, position) -> doubleSet.contains(type.getDouble(column, position)); + }; + case BOOLEAN -> + values -> { + Set booleanSet = new HashSet<>(); + for (String value : values) { + if ("true".equalsIgnoreCase(value)) { + booleanSet.add(true); + } else if ("false".equalsIgnoreCase(value)) { + booleanSet.add(false); + } else { + throw new SemanticException( + String.format(CalcMessages.CANNOT_CAST_TO_BOOLEAN, value)); + } + } + return (column, position) -> + booleanSet.contains(type.getBoolean(column, position)); + }; + case TEXT, STRING -> + values -> { + Set stringSet = + values.stream() + .map(value -> new Binary(value, TSFileConfig.STRING_CHARSET)) + .collect(Collectors.toSet()); + return (column, position) -> stringSet.contains(type.getBinary(column, position)); + }; + case BLOB, DATE, OBJECT, ROW, UNKNOWN, VECTOR -> + values -> { + throw new UnsupportedOperationException( + CalcMessages.UNSUPPORTED_DATA_TYPE_LOWER + type.getTypeEnum()); + }; + }; + + public static final TypeService + IN_MULTI_COLUMN_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (valueColumnTransformers, values) -> { + Set intSet = new HashSet<>(); + for (Literal value : values) { + try { + long v = ((LongLiteral) value).getParsedValue(); + if (v <= Integer.MAX_VALUE && v >= Integer.MIN_VALUE) { + intSet.add((int) v); + } + } catch (IllegalArgumentException e) { + throw new SemanticException( + String.format( + CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum())); + } + } + return new InInt32MultiColumnTransformer(intSet, valueColumnTransformers); + }; + case DATE -> + (valueColumnTransformers, values) -> { + Set dateSet = new HashSet<>(); + for (Literal value : values) { + dateSet.add(Integer.parseInt(((GenericLiteral) value).getValue())); + } + return new InInt32MultiColumnTransformer(dateSet, valueColumnTransformers); + }; + case INT64 -> + (valueColumnTransformers, values) -> { + Set longSet = new HashSet<>(); + for (Literal value : values) { + longSet.add(((LongLiteral) value).getParsedValue()); + } + return new InInt64MultiColumnTransformer(longSet, valueColumnTransformers); + }; + case TIMESTAMP -> + (valueColumnTransformers, values) -> { + Set timestampSet = new HashSet<>(); + for (Literal value : values) { + try { + if (value instanceof LongLiteral) { + timestampSet.add(((LongLiteral) value).getParsedValue()); + } else if (value instanceof DoubleLiteral) { + timestampSet.add((long) ((DoubleLiteral) value).getValue()); + } else if (value instanceof FloatLiteral) { + timestampSet.add((long) ((FloatLiteral) value).getValue()); + } else if (value instanceof GenericLiteral) { + timestampSet.add(Long.parseLong(((GenericLiteral) value).getValue())); + } else { + throw new SemanticException( + String.format( + CalcMessages.IN_LIST_LITERAL_FOR_TIMESTAMP_TYPE_RESTRICTION, + value.getClass().getSimpleName())); + } + } catch (IllegalArgumentException e) { + throw new SemanticException( + String.format( + CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum())); + } + } + return new InInt64MultiColumnTransformer( + timestampSet, valueColumnTransformers); + }; + case FLOAT -> + (valueColumnTransformers, values) -> { + Set floatSet = new HashSet<>(); + for (Literal value : values) { + try { + if (value instanceof FloatLiteral) { + floatSet.add(((FloatLiteral) value).getValue()); + } else { + floatSet.add((float) ((DoubleLiteral) value).getValue()); + } + } catch (IllegalArgumentException e) { + throw new SemanticException( + String.format( + CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum())); + } + } + return new InFloatMultiColumnTransformer(floatSet, valueColumnTransformers); + }; + case DOUBLE -> + (valueColumnTransformers, values) -> { + Set doubleSet = new HashSet<>(); + for (Literal value : values) { + try { + if (value instanceof FloatLiteral) { + doubleSet.add((double) ((FloatLiteral) value).getValue()); + } else { + doubleSet.add(((DoubleLiteral) value).getValue()); + } + } catch (IllegalArgumentException e) { + throw new SemanticException( + String.format( + CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum())); + } + } + return new InDoubleMultiColumnTransformer(doubleSet, valueColumnTransformers); + }; + case BOOLEAN -> + (valueColumnTransformers, values) -> { + Set booleanSet = new HashSet<>(); + for (Literal value : values) { + booleanSet.add(((BooleanLiteral) value).getValue()); + } + return new InBooleanMultiColumnTransformer( + booleanSet, valueColumnTransformers); + }; + case TEXT, STRING -> + (valueColumnTransformers, values) -> { + Set stringSet = new HashSet<>(); + for (Literal value : values) { + stringSet.add( + new Binary( + ((StringLiteral) value).getValue(), TSFileConfig.STRING_CHARSET)); + } + return new InBinaryMultiColumnTransformer(stringSet, valueColumnTransformers); + }; + case BLOB -> + (valueColumnTransformers, values) -> { + Set binarySet = new HashSet<>(); + for (Literal value : values) { + binarySet.add(new Binary(((BinaryLiteral) value).getValue())); + } + return new InBinaryMultiColumnTransformer(binarySet, valueColumnTransformers); + }; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (valueColumnTransformers, values) -> { + throw new UnsupportedOperationException( + CalcMessages.UNSUPPORTED_DATA_TYPE_LOWER + type.getTypeEnum()); + }; + }; + + /** Updates a result with the minimum value from statistics. */ + public static final TypeService MIN_STATISTICS_VALUE_UPDATER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (result, value, initialized, exceptionSupplier) -> { + int candidate = ((Number) value).intValue(); + if (!initialized || candidate < result.getInt()) { + result.setInt(candidate); + return true; + } + return false; + }; + case INT64, TIMESTAMP -> + (result, value, initialized, exceptionSupplier) -> { + long candidate = ((Number) value).longValue(); + if (!initialized || candidate < result.getLong()) { + result.setLong(candidate); + return true; + } + return false; + }; + case FLOAT -> + (result, value, initialized, exceptionSupplier) -> { + float candidate = ((Number) value).floatValue(); + if (!initialized || candidate < result.getFloat()) { + result.setFloat(candidate); + return true; + } + return false; + }; + case DOUBLE -> + (result, value, initialized, exceptionSupplier) -> { + double candidate = ((Number) value).doubleValue(); + if (!initialized || candidate < result.getDouble()) { + result.setDouble(candidate); + return true; + } + return false; + }; + case BOOLEAN -> + (result, value, initialized, exceptionSupplier) -> { + boolean candidate = (boolean) value; + if (!initialized || !candidate) { + result.setBoolean(candidate); + return true; + } + return false; + }; + case TEXT, STRING, BLOB -> + (result, value, initialized, exceptionSupplier) -> { + Binary candidate = toBinary(value); + if (!initialized || candidate.compareTo(result.getBinary()) < 0) { + result.setBinary(candidate); + return true; + } + return false; + }; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (result, value, initialized, exceptionSupplier) -> { + throw exceptionSupplier.get(); + }; + }; + + /** Updates a result with the maximum value from statistics. */ + public static final TypeService MAX_STATISTICS_VALUE_UPDATER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (result, value, initialized, exceptionSupplier) -> { + int candidate = ((Number) value).intValue(); + if (!initialized || candidate > result.getInt()) { + result.setInt(candidate); + return true; + } + return false; + }; + case INT64, TIMESTAMP -> + (result, value, initialized, exceptionSupplier) -> { + long candidate = ((Number) value).longValue(); + if (!initialized || candidate > result.getLong()) { + result.setLong(candidate); + return true; + } + return false; + }; + case FLOAT -> + (result, value, initialized, exceptionSupplier) -> { + float candidate = ((Number) value).floatValue(); + if (!initialized || candidate > result.getFloat()) { + result.setFloat(candidate); + return true; + } + return false; + }; + case DOUBLE -> + (result, value, initialized, exceptionSupplier) -> { + double candidate = ((Number) value).doubleValue(); + if (!initialized || candidate > result.getDouble()) { + result.setDouble(candidate); + return true; + } + return false; + }; + case BOOLEAN -> + (result, value, initialized, exceptionSupplier) -> { + boolean candidate = (boolean) value; + if (!initialized || candidate) { + result.setBoolean(candidate); + return true; + } + return false; + }; + case TEXT, STRING, BLOB -> + (result, value, initialized, exceptionSupplier) -> { + Binary candidate = toBinary(value); + if (!initialized || candidate.compareTo(result.getBinary()) > 0) { + result.setBinary(candidate); + return true; + } + return false; + }; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (result, value, initialized, exceptionSupplier) -> { + throw exceptionSupplier.get(); + }; + }; + + /** Updates a result with the greatest absolute numeric value from statistics. */ + public static final TypeService EXTREME_STATISTICS_VALUE_UPDATER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (result, value, initialized, exceptionSupplier) -> { + int candidate = ((Number) value).intValue(); + if (!initialized || compareExtreme(candidate, result.getInt()) > 0) { + result.setInt(candidate); + return true; + } + return false; + }; + case INT64 -> + (result, value, initialized, exceptionSupplier) -> { + long candidate = ((Number) value).longValue(); + if (!initialized || compareExtreme(candidate, result.getLong()) > 0) { + result.setLong(candidate); + return true; + } + return false; + }; + case FLOAT -> + (result, value, initialized, exceptionSupplier) -> { + float candidate = ((Number) value).floatValue(); + float absCandidate = Math.abs(candidate); + float absResult = Math.abs(result.getFloat()); + if (!initialized + || absCandidate > absResult + || absCandidate == absResult && candidate > result.getFloat()) { + result.setFloat(candidate); + return true; + } + return false; + }; + case DOUBLE -> + (result, value, initialized, exceptionSupplier) -> { + double candidate = ((Number) value).doubleValue(); + double absCandidate = Math.abs(candidate); + double absResult = Math.abs(result.getDouble()); + if (!initialized + || absCandidate > absResult + || absCandidate == absResult && candidate > result.getDouble()) { + result.setDouble(candidate); + return true; + } + return false; + }; + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (result, value, initialized, exceptionSupplier) -> { + throw exceptionSupplier.get(); + }; + }; + + /** Binds an APPROX_PERCENTILE input reader to the input column's native type. */ + public static final TypeService< + Function>> + APPROX_PERCENTILE_INPUT_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> accumulator -> accumulator::addIntInput; + case INT64, TIMESTAMP -> accumulator -> accumulator::addLongInput; + case FLOAT -> accumulator -> accumulator::addFloatInput; + case DOUBLE -> accumulator -> accumulator::addDoubleInput; + case BOOLEAN, DATE, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + accumulator -> + (arguments, mask) -> { + throw new UnSupportedDataTypeException( + String.format( + CalcMessages + .UNSUPPORTED_DATA_TYPE_IN_APPROX_PERCENTILE_AGGREGATION, + type.getTypeEnum())); + }; + }; + + /** Binds a grouped APPROX_PERCENTILE input reader to the input column's native type. */ + public static final TypeService< + Function< + AbstractGroupedApproxPercentileAccumulator, + TriConsumer>> + GROUPED_APPROX_PERCENTILE_INPUT_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> accumulator -> accumulator::addIntInput; + case INT64, TIMESTAMP -> accumulator -> accumulator::addLongInput; + case FLOAT -> accumulator -> accumulator::addFloatInput; + case DOUBLE -> accumulator -> accumulator::addDoubleInput; + case BOOLEAN, DATE, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + accumulator -> + (groupIds, arguments, mask) -> { + throw new UnSupportedDataTypeException( + String.format( + CalcMessages + .UNSUPPORTED_DATA_TYPE_IN_APPROX_PERCENTILE_AGGREGATION, + type.getTypeEnum())); + }; + }; + + /** Provides storage operations for grouped accumulators backed by primitive big arrays. */ + public static final TypeService GROUPED_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + new GroupedValueService() { + @Override + public void initialize(GroupedValueAccessor accessor) { + accessor.initializeIntValues(); + } + + @Override + public void ensureCapacity(GroupedValueAccessor accessor, long groupCount) { + accessor.getIntValues().ensureCapacity(groupCount); + } + + @Override + public long sizeOf(GroupedValueAccessor accessor) { + return accessor.getIntValues().sizeOf(); + } + + @Override + public void reset(GroupedValueAccessor accessor) { + accessor.getIntValues().reset(); + } + + @Override + public void write( + GroupedValueAccessor accessor, int groupId, ColumnBuilder columnBuilder) { + columnBuilder.writeInt(accessor.getIntValues().get(groupId)); + } + }; + case INT64, TIMESTAMP -> + new GroupedValueService() { + @Override + public void initialize(GroupedValueAccessor accessor) { + accessor.initializeLongValues(); + } + + @Override + public void ensureCapacity(GroupedValueAccessor accessor, long groupCount) { + accessor.getLongValues().ensureCapacity(groupCount); + } + + @Override + public long sizeOf(GroupedValueAccessor accessor) { + return accessor.getLongValues().sizeOf(); + } + + @Override + public void reset(GroupedValueAccessor accessor) { + accessor.getLongValues().reset(); + } + + @Override + public void write( + GroupedValueAccessor accessor, int groupId, ColumnBuilder columnBuilder) { + columnBuilder.writeLong(accessor.getLongValues().get(groupId)); + } + }; + case FLOAT -> + new GroupedValueService() { + @Override + public void initialize(GroupedValueAccessor accessor) { + accessor.initializeFloatValues(); + } + + @Override + public void ensureCapacity(GroupedValueAccessor accessor, long groupCount) { + accessor.getFloatValues().ensureCapacity(groupCount); + } + + @Override + public long sizeOf(GroupedValueAccessor accessor) { + return accessor.getFloatValues().sizeOf(); + } + + @Override + public void reset(GroupedValueAccessor accessor) { + accessor.getFloatValues().reset(); + } + + @Override + public void write( + GroupedValueAccessor accessor, int groupId, ColumnBuilder columnBuilder) { + columnBuilder.writeFloat(accessor.getFloatValues().get(groupId)); + } + }; + case DOUBLE -> + new GroupedValueService() { + @Override + public void initialize(GroupedValueAccessor accessor) { + accessor.initializeDoubleValues(); + } + + @Override + public void ensureCapacity(GroupedValueAccessor accessor, long groupCount) { + accessor.getDoubleValues().ensureCapacity(groupCount); + } + + @Override + public long sizeOf(GroupedValueAccessor accessor) { + return accessor.getDoubleValues().sizeOf(); + } + + @Override + public void reset(GroupedValueAccessor accessor) { + accessor.getDoubleValues().reset(); + } + + @Override + public void write( + GroupedValueAccessor accessor, int groupId, ColumnBuilder columnBuilder) { + columnBuilder.writeDouble(accessor.getDoubleValues().get(groupId)); + } + }; + case TEXT, STRING, BLOB, OBJECT -> + new GroupedValueService() { + @Override + public void initialize(GroupedValueAccessor accessor) { + accessor.initializeBinaryValues(); + } + + @Override + public void ensureCapacity(GroupedValueAccessor accessor, long groupCount) { + accessor.getBinaryValues().ensureCapacity(groupCount); + } + + @Override + public long sizeOf(GroupedValueAccessor accessor) { + return accessor.getBinaryValues().sizeOf(); + } + + @Override + public void reset(GroupedValueAccessor accessor) { + accessor.getBinaryValues().reset(); + } + + @Override + public void write( + GroupedValueAccessor accessor, int groupId, ColumnBuilder columnBuilder) { + columnBuilder.writeBinary(accessor.getBinaryValues().get(groupId)); + } + }; + case BOOLEAN -> + new GroupedValueService() { + @Override + public void initialize(GroupedValueAccessor accessor) { + accessor.initializeBooleanValues(); + } + + @Override + public void ensureCapacity(GroupedValueAccessor accessor, long groupCount) { + accessor.getBooleanValues().ensureCapacity(groupCount); + } + + @Override + public long sizeOf(GroupedValueAccessor accessor) { + return accessor.getBooleanValues().sizeOf(); + } + + @Override + public void reset(GroupedValueAccessor accessor) { + accessor.getBooleanValues().reset(); + } + + @Override + public void write( + GroupedValueAccessor accessor, int groupId, ColumnBuilder columnBuilder) { + columnBuilder.writeBoolean(accessor.getBooleanValues().get(groupId)); + } + }; + case ROW, UNKNOWN, VECTOR -> unsupportedGroupedValueService(type); + }; + + /** Reads grouped input and intermediate columns using the native column accessor. */ + public static final TypeService GROUPED_INPUT_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + new GroupedInputReader() { + @Override + public void addInput( + int[] groupIds, + Column[] arguments, + AggregationMask mask, + GroupedInputConsumer consumer) { + forEachSelectedPosition( + arguments[0], + mask, + position -> + consumer.updateInt(groupIds[position], arguments[0].getInt(position))); + } + + @Override + public void addIntermediate( + int[] groupIds, Column argument, GroupedInputConsumer consumer) { + for (int i = 0; i < groupIds.length; i++) { + if (!argument.isNull(i)) { + consumer.updateInt(groupIds[i], argument.getInt(i)); + } + } + } + }; + case INT64, TIMESTAMP -> + new GroupedInputReader() { + @Override + public void addInput( + int[] groupIds, + Column[] arguments, + AggregationMask mask, + GroupedInputConsumer consumer) { + forEachSelectedPosition( + arguments[0], + mask, + position -> + consumer.updateLong( + groupIds[position], arguments[0].getLong(position))); + } + + @Override + public void addIntermediate( + int[] groupIds, Column argument, GroupedInputConsumer consumer) { + for (int i = 0; i < groupIds.length; i++) { + if (!argument.isNull(i)) { + consumer.updateLong(groupIds[i], argument.getLong(i)); + } + } + } + }; + case FLOAT -> + new GroupedInputReader() { + @Override + public void addInput( + int[] groupIds, + Column[] arguments, + AggregationMask mask, + GroupedInputConsumer consumer) { + forEachSelectedPosition( + arguments[0], + mask, + position -> + consumer.updateFloat( + groupIds[position], arguments[0].getFloat(position))); + } + + @Override + public void addIntermediate( + int[] groupIds, Column argument, GroupedInputConsumer consumer) { + for (int i = 0; i < groupIds.length; i++) { + if (!argument.isNull(i)) { + consumer.updateFloat(groupIds[i], argument.getFloat(i)); + } + } + } + }; + case DOUBLE -> + new GroupedInputReader() { + @Override + public void addInput( + int[] groupIds, + Column[] arguments, + AggregationMask mask, + GroupedInputConsumer consumer) { + forEachSelectedPosition( + arguments[0], + mask, + position -> + consumer.updateDouble( + groupIds[position], arguments[0].getDouble(position))); + } + + @Override + public void addIntermediate( + int[] groupIds, Column argument, GroupedInputConsumer consumer) { + for (int i = 0; i < groupIds.length; i++) { + if (!argument.isNull(i)) { + consumer.updateDouble(groupIds[i], argument.getDouble(i)); + } + } + } + }; + case TEXT, STRING, BLOB, OBJECT -> + new GroupedInputReader() { + @Override + public void addInput( + int[] groupIds, + Column[] arguments, + AggregationMask mask, + GroupedInputConsumer consumer) { + forEachSelectedPosition( + arguments[0], + mask, + position -> + consumer.updateBinary( + groupIds[position], arguments[0].getBinary(position))); + } + + @Override + public void addIntermediate( + int[] groupIds, Column argument, GroupedInputConsumer consumer) { + for (int i = 0; i < groupIds.length; i++) { + if (!argument.isNull(i)) { + consumer.updateBinary(groupIds[i], argument.getBinary(i)); + } + } + } + }; + case BOOLEAN -> + new GroupedInputReader() { + @Override + public void addInput( + int[] groupIds, + Column[] arguments, + AggregationMask mask, + GroupedInputConsumer consumer) { + forEachSelectedPosition( + arguments[0], + mask, + position -> + consumer.updateBoolean( + groupIds[position], arguments[0].getBoolean(position))); + } + + @Override + public void addIntermediate( + int[] groupIds, Column argument, GroupedInputConsumer consumer) { + for (int i = 0; i < groupIds.length; i++) { + if (!argument.isNull(i)) { + consumer.updateBoolean(groupIds[i], argument.getBoolean(i)); + } + } + } + }; + case ROW, UNKNOWN, VECTOR -> unsupportedGroupedInputReader(type); + }; + + /** Restricts grouped EXTREME to the numeric types supported by the aggregate. */ + public static final TypeService GROUPED_EXTREME_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, INT64, FLOAT, DOUBLE -> GROUPED_VALUE_SERVICE.call(type); + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + unsupportedGroupedValueService(type); + }; + + public static final TypeService GROUPED_EXTREME_INPUT_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, INT64, FLOAT, DOUBLE -> GROUPED_INPUT_READER_SERVICE.call(type); + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + unsupportedGroupedInputReader(type); + }; + + /** Reads a value column together with its optional ordering-time column. */ + public static final TypeService GROUPED_TIME_INPUT_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (groupIds, arguments, mask, consumer) -> + forEachSelectedPosition( + arguments[0], + mask, + position -> + consumer.updateInt( + groupIds[position], + arguments[0].getInt(position), + arguments[1].isNull(position), + arguments[1].isNull(position) + ? 0 + : arguments[1].getLong(position))); + case INT64, TIMESTAMP -> + (groupIds, arguments, mask, consumer) -> + forEachSelectedPosition( + arguments[0], + mask, + position -> + consumer.updateLong( + groupIds[position], + arguments[0].getLong(position), + arguments[1].isNull(position), + arguments[1].isNull(position) + ? 0 + : arguments[1].getLong(position))); + case FLOAT -> + (groupIds, arguments, mask, consumer) -> + forEachSelectedPosition( + arguments[0], + mask, + position -> + consumer.updateFloat( + groupIds[position], + arguments[0].getFloat(position), + arguments[1].isNull(position), + arguments[1].isNull(position) + ? 0 + : arguments[1].getLong(position))); + case DOUBLE -> + (groupIds, arguments, mask, consumer) -> + forEachSelectedPosition( + arguments[0], + mask, + position -> + consumer.updateDouble( + groupIds[position], + arguments[0].getDouble(position), + arguments[1].isNull(position), + arguments[1].isNull(position) + ? 0 + : arguments[1].getLong(position))); + case TEXT, STRING, BLOB, OBJECT -> + (groupIds, arguments, mask, consumer) -> + forEachSelectedPosition( + arguments[0], + mask, + position -> + consumer.updateBinary( + groupIds[position], + arguments[0].getBinary(position), + arguments[1].isNull(position), + arguments[1].isNull(position) + ? 0 + : arguments[1].getLong(position))); + case BOOLEAN -> + (groupIds, arguments, mask, consumer) -> + forEachSelectedPosition( + arguments[0], + mask, + position -> + consumer.updateBoolean( + groupIds[position], + arguments[0].getBoolean(position), + arguments[1].isNull(position), + arguments[1].isNull(position) + ? 0 + : arguments[1].getLong(position))); + case ROW, UNKNOWN, VECTOR -> + (groupIds, arguments, mask, consumer) -> { + throw unsupportedDataType(type); + }; + }; + + /** Reads x values for FIRST_BY/LAST_BY while filtering null y values. */ + public static final TypeService GROUPED_BY_INPUT_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (groupIds, arguments, mask, consumer) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + consumer.updateInt( + groupIds[position], + arguments[0].isNull(position), + arguments[0].getInt(position), + arguments[2].isNull(position), + arguments[2].isNull(position) + ? 0 + : arguments[2].getLong(position))); + case INT64, TIMESTAMP -> + (groupIds, arguments, mask, consumer) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + consumer.updateLong( + groupIds[position], + arguments[0].isNull(position), + arguments[0].getLong(position), + arguments[2].isNull(position), + arguments[2].isNull(position) + ? 0 + : arguments[2].getLong(position))); + case FLOAT -> + (groupIds, arguments, mask, consumer) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + consumer.updateFloat( + groupIds[position], + arguments[0].isNull(position), + arguments[0].getFloat(position), + arguments[2].isNull(position), + arguments[2].isNull(position) + ? 0 + : arguments[2].getLong(position))); + case DOUBLE -> + (groupIds, arguments, mask, consumer) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + consumer.updateDouble( + groupIds[position], + arguments[0].isNull(position), + arguments[0].getDouble(position), + arguments[2].isNull(position), + arguments[2].isNull(position) + ? 0 + : arguments[2].getLong(position))); + case TEXT, STRING, BLOB, OBJECT -> + (groupIds, arguments, mask, consumer) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + consumer.updateBinary( + groupIds[position], + arguments[0].isNull(position), + arguments[0].getBinary(position), + arguments[2].isNull(position), + arguments[2].isNull(position) + ? 0 + : arguments[2].getLong(position))); + case BOOLEAN -> + (groupIds, arguments, mask, consumer) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + consumer.updateBoolean( + groupIds[position], + arguments[0].isNull(position), + arguments[0].getBoolean(position), + arguments[2].isNull(position), + arguments[2].isNull(position) + ? 0 + : arguments[2].getLong(position))); + case ROW, UNKNOWN, VECTOR -> + (groupIds, arguments, mask, consumer) -> { + throw unsupportedDataType(type); + }; + }; + + /** Serializes/deserializes a grouped value in the native intermediate format. */ + public static final TypeService GROUPED_VALUE_SERIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + new GroupedValueSerializer() { + @Override + public int calcTypeSize(GroupedValueAccessor accessor, long index) { + return Integer.BYTES; + } + + @Override + public void serialize( + GroupedValueAccessor accessor, long index, byte[] bytes, int offset) { + accessor.getIntValues().toBytes(index, bytes, offset); + } + }; + case INT64, TIMESTAMP -> + new GroupedValueSerializer() { + @Override + public int calcTypeSize(GroupedValueAccessor accessor, long index) { + return Long.BYTES; + } + + @Override + public void serialize( + GroupedValueAccessor accessor, long index, byte[] bytes, int offset) { + accessor.getLongValues().toBytes(index, bytes, offset); + } + }; + case FLOAT -> + new GroupedValueSerializer() { + @Override + public int calcTypeSize(GroupedValueAccessor accessor, long index) { + return Float.BYTES; + } + + @Override + public void serialize( + GroupedValueAccessor accessor, long index, byte[] bytes, int offset) { + accessor.getFloatValues().toBytes(index, bytes, offset); + } + }; + case DOUBLE -> + new GroupedValueSerializer() { + @Override + public int calcTypeSize(GroupedValueAccessor accessor, long index) { + return Double.BYTES; + } + + @Override + public void serialize( + GroupedValueAccessor accessor, long index, byte[] bytes, int offset) { + accessor.getDoubleValues().toBytes(index, bytes, offset); + } + }; + case TEXT, STRING, BLOB, OBJECT -> + new GroupedValueSerializer() { + @Override + public int calcTypeSize(GroupedValueAccessor accessor, long index) { + return Integer.BYTES + accessor.getBinaryValues().get(index).getLength(); + } + + @Override + public void serialize( + GroupedValueAccessor accessor, long index, byte[] bytes, int offset) { + accessor.getBinaryValues().toBytes(index, bytes, offset); + } + }; + case BOOLEAN -> + new GroupedValueSerializer() { + @Override + public int calcTypeSize(GroupedValueAccessor accessor, long index) { + return Byte.BYTES; + } + + @Override + public void serialize( + GroupedValueAccessor accessor, long index, byte[] bytes, int offset) { + BytesUtils.boolToBytes(accessor.getBooleanValues().get(index), bytes, offset); + } + }; + case ROW, UNKNOWN, VECTOR -> unsupportedGroupedValueSerializer(type); + }; + + /** Stores a column value in the typed big array selected for a grouped accumulator. */ + public static final TypeService GROUPED_VALUE_SETTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (accessor, index, column, position) -> + accessor.getIntValues().set(index, type.getInt(column, position)); + case INT64, TIMESTAMP -> + (accessor, index, column, position) -> + accessor.getLongValues().set(index, type.getLong(column, position)); + case FLOAT -> + (accessor, index, column, position) -> + accessor.getFloatValues().set(index, type.getFloat(column, position)); + case DOUBLE -> + (accessor, index, column, position) -> + accessor.getDoubleValues().set(index, type.getDouble(column, position)); + case TEXT, STRING, BLOB, OBJECT -> + (accessor, index, column, position) -> + accessor.getBinaryValues().set(index, type.getBinary(column, position)); + case BOOLEAN -> + (accessor, index, column, position) -> + accessor.getBooleanValues().set(index, type.getBoolean(column, position)); + case ROW, UNKNOWN, VECTOR -> + (accessor, index, column, position) -> { + throw unsupportedDataType(type); + }; + }; + + /** Reads a grouped value from bytes into a typed callback. */ + public static final TypeService + GROUPED_TIME_VALUE_DESERIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (bytes, offset, groupId, time, timeNull, consumer) -> + consumer.updateInt( + groupId, BytesUtils.bytesToInt(bytes, offset), timeNull, time); + case INT64, TIMESTAMP -> + (bytes, offset, groupId, time, timeNull, consumer) -> + consumer.updateLong( + groupId, + BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset), + timeNull, + time); + case FLOAT -> + (bytes, offset, groupId, time, timeNull, consumer) -> + consumer.updateFloat( + groupId, BytesUtils.bytesToFloat(bytes, offset), timeNull, time); + case DOUBLE -> + (bytes, offset, groupId, time, timeNull, consumer) -> + consumer.updateDouble( + groupId, BytesUtils.bytesToDouble(bytes, offset), timeNull, time); + case TEXT, STRING, BLOB, OBJECT -> + (bytes, offset, groupId, time, timeNull, consumer) -> { + int length = BytesUtils.bytesToInt(bytes, offset); + consumer.updateBinary( + groupId, + new Binary(BytesUtils.subBytes(bytes, offset + Integer.BYTES, length)), + timeNull, + time); + }; + case BOOLEAN -> + (bytes, offset, groupId, time, timeNull, consumer) -> + consumer.updateBoolean( + groupId, BytesUtils.bytesToBool(bytes, offset), timeNull, time); + case ROW, UNKNOWN, VECTOR -> + (bytes, offset, groupId, time, timeNull, consumer) -> { + throw unsupportedDataType(type); + }; + }; + + /** Reads x values from FIRST_BY/LAST_BY intermediate bytes. */ + public static final TypeService + GROUPED_BY_VALUE_DESERIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (bytes, offset, groupId, xNull, time, timeNull, consumer) -> + consumer.updateInt( + groupId, + xNull, + xNull ? 0 : BytesUtils.bytesToInt(bytes, offset), + timeNull, + time); + case INT64, TIMESTAMP -> + (bytes, offset, groupId, xNull, time, timeNull, consumer) -> + consumer.updateLong( + groupId, + xNull, + xNull ? 0 : BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset), + timeNull, + time); + case FLOAT -> + (bytes, offset, groupId, xNull, time, timeNull, consumer) -> + consumer.updateFloat( + groupId, + xNull, + xNull ? 0 : BytesUtils.bytesToFloat(bytes, offset), + timeNull, + time); + case DOUBLE -> + (bytes, offset, groupId, xNull, time, timeNull, consumer) -> + consumer.updateDouble( + groupId, + xNull, + xNull ? 0 : BytesUtils.bytesToDouble(bytes, offset), + timeNull, + time); + case TEXT, STRING, BLOB, OBJECT -> + (bytes, offset, groupId, xNull, time, timeNull, consumer) -> { + Binary value = null; + if (!xNull) { + int length = BytesUtils.bytesToInt(bytes, offset); + value = + new Binary(BytesUtils.subBytes(bytes, offset + Integer.BYTES, length)); + } + consumer.updateBinary(groupId, xNull, value, timeNull, time); + }; + case BOOLEAN -> + (bytes, offset, groupId, xNull, time, timeNull, consumer) -> + consumer.updateBoolean( + groupId, + xNull, + xNull ? false : BytesUtils.bytesToBool(bytes, offset), + timeNull, + time); + case ROW, UNKNOWN, VECTOR -> + (bytes, offset, groupId, xNull, time, timeNull, consumer) -> { + throw unsupportedDataType(type); + }; + }; + + /** Selects the y-column reader used by MAX_BY and MIN_BY input processing. */ + public static final TypeService MAX_MIN_BY_INPUT_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (arguments, mask, updater) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + updater.updateInt( + arguments[1].getInt(position), arguments[0], position)); + case INT64, TIMESTAMP -> + (arguments, mask, updater) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + updater.updateLong( + arguments[1].getLong(position), arguments[0], position)); + case FLOAT -> + (arguments, mask, updater) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + updater.updateFloat( + arguments[1].getFloat(position), arguments[0], position)); + case DOUBLE -> + (arguments, mask, updater) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + updater.updateDouble( + arguments[1].getDouble(position), arguments[0], position)); + case TEXT, STRING, BLOB, OBJECT -> + (arguments, mask, updater) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + updater.updateBinary( + arguments[1].getBinary(position), arguments[0], position)); + case BOOLEAN -> + (arguments, mask, updater) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + updater.updateBoolean( + arguments[1].getBoolean(position), arguments[0], position)); + case ROW, UNKNOWN, VECTOR -> + (arguments, mask, updater) -> { + throw unsupportedMaxMinByDataType(type); + }; + }; + + /** Selects the y-value decoder used by MAX_BY and MIN_BY intermediate input processing. */ + public static final TypeService MAX_MIN_BY_INTERMEDIATE_INPUT_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (bytes, xType, columnBuilder, updater) -> { + int value = BytesUtils.bytesToInt(bytes, 0); + readMaxMinByXFromBytes(bytes, Integer.BYTES, xType, columnBuilder); + updater.updateInt(value, columnBuilder.build(), 0); + }; + case INT64, TIMESTAMP -> + (bytes, xType, columnBuilder, updater) -> { + long value = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); + readMaxMinByXFromBytes(bytes, Long.BYTES, xType, columnBuilder); + updater.updateLong(value, columnBuilder.build(), 0); + }; + case FLOAT -> + (bytes, xType, columnBuilder, updater) -> { + float value = BytesUtils.bytesToFloat(bytes, 0); + readMaxMinByXFromBytes(bytes, Float.BYTES, xType, columnBuilder); + updater.updateFloat(value, columnBuilder.build(), 0); + }; + case DOUBLE -> + (bytes, xType, columnBuilder, updater) -> { + double value = BytesUtils.bytesToDouble(bytes, 0); + readMaxMinByXFromBytes(bytes, Double.BYTES, xType, columnBuilder); + updater.updateDouble(value, columnBuilder.build(), 0); + }; + case TEXT, STRING, BLOB, OBJECT -> + (bytes, xType, columnBuilder, updater) -> { + int length = BytesUtils.bytesToInt(bytes, 0); + int xOffset = Integer.BYTES + length; + Binary value = new Binary(BytesUtils.subBytes(bytes, Integer.BYTES, length)); + readMaxMinByXFromBytes(bytes, xOffset, xType, columnBuilder); + updater.updateBinary(value, columnBuilder.build(), 0); + }; + case BOOLEAN -> + (bytes, xType, columnBuilder, updater) -> { + boolean value = BytesUtils.bytesToBool(bytes, 0); + readMaxMinByXFromBytes(bytes, 1, xType, columnBuilder); + updater.updateBoolean(value, columnBuilder.build(), 0); + }; + case ROW, UNKNOWN, VECTOR -> + (bytes, xType, columnBuilder, updater) -> { + throw unsupportedMaxMinByDataType(type); + }; + }; + + /** Selects the y-column reader used by grouped MAX_BY and MIN_BY input processing. */ + public static final TypeService GROUPED_MAX_MIN_BY_INPUT_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (groupIds, arguments, mask, updater) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + updater.updateInt( + groupIds[position], + arguments[1].getInt(position), + arguments[0], + position)); + case INT64, TIMESTAMP -> + (groupIds, arguments, mask, updater) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + updater.updateLong( + groupIds[position], + arguments[1].getLong(position), + arguments[0], + position)); + case FLOAT -> + (groupIds, arguments, mask, updater) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + updater.updateFloat( + groupIds[position], + arguments[1].getFloat(position), + arguments[0], + position)); + case DOUBLE -> + (groupIds, arguments, mask, updater) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + updater.updateDouble( + groupIds[position], + arguments[1].getDouble(position), + arguments[0], + position)); + case TEXT, STRING, BLOB, OBJECT -> + (groupIds, arguments, mask, updater) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + updater.updateBinary( + groupIds[position], + arguments[1].getBinary(position), + arguments[0], + position)); + case BOOLEAN -> + (groupIds, arguments, mask, updater) -> + forEachSelectedPosition( + arguments[1], + mask, + position -> + updater.updateBoolean( + groupIds[position], + arguments[1].getBoolean(position), + arguments[0], + position)); + case ROW, UNKNOWN, VECTOR -> + (groupIds, arguments, mask, updater) -> { + throw unsupportedMaxMinByDataType(type); + }; + }; + + /** Selects the y-value decoder used by grouped MAX_BY and MIN_BY intermediate input. */ + public static final TypeService + GROUPED_MAX_MIN_BY_INTERMEDIATE_INPUT_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (groupId, bytes, xType, columnBuilder, updater) -> { + int value = BytesUtils.bytesToInt(bytes, 0); + readMaxMinByXFromBytes(bytes, Integer.BYTES, xType, columnBuilder); + updater.updateInt(groupId, value, columnBuilder.build(), 0); + }; + case INT64, TIMESTAMP -> + (groupId, bytes, xType, columnBuilder, updater) -> { + long value = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); + readMaxMinByXFromBytes(bytes, Long.BYTES, xType, columnBuilder); + updater.updateLong(groupId, value, columnBuilder.build(), 0); + }; + case FLOAT -> + (groupId, bytes, xType, columnBuilder, updater) -> { + float value = BytesUtils.bytesToFloat(bytes, 0); + readMaxMinByXFromBytes(bytes, Float.BYTES, xType, columnBuilder); + updater.updateFloat(groupId, value, columnBuilder.build(), 0); + }; + case DOUBLE -> + (groupId, bytes, xType, columnBuilder, updater) -> { + double value = BytesUtils.bytesToDouble(bytes, 0); + readMaxMinByXFromBytes(bytes, Double.BYTES, xType, columnBuilder); + updater.updateDouble(groupId, value, columnBuilder.build(), 0); + }; + case TEXT, STRING, BLOB, OBJECT -> + (groupId, bytes, xType, columnBuilder, updater) -> { + int length = BytesUtils.bytesToInt(bytes, 0); + int xOffset = Integer.BYTES + length; + Binary value = new Binary(BytesUtils.subBytes(bytes, Integer.BYTES, length)); + readMaxMinByXFromBytes(bytes, xOffset, xType, columnBuilder); + updater.updateBinary(groupId, value, columnBuilder.build(), 0); + }; + case BOOLEAN -> + (groupId, bytes, xType, columnBuilder, updater) -> { + boolean value = BytesUtils.bytesToBool(bytes, 0); + readMaxMinByXFromBytes(bytes, Byte.BYTES, xType, columnBuilder); + updater.updateBoolean(groupId, value, columnBuilder.build(), 0); + }; + case ROW, UNKNOWN, VECTOR -> + (groupId, bytes, xType, columnBuilder, updater) -> { + throw unsupportedMaxMinByDataType(type); + }; + }; + + /** Selects the type-specific decoder for a LAST intermediate value. */ + public static final TypeService LAST_VALUE_DESERIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (bytes, offset, time, isOrderTimeNull, updater) -> { + int value = BytesUtils.bytesToInt(bytes, offset); + updater.updateInt(value, time, isOrderTimeNull); + }; + case INT64, TIMESTAMP -> + (bytes, offset, time, isOrderTimeNull, updater) -> { + long value = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); + updater.updateLong(value, time, isOrderTimeNull); + }; + case FLOAT -> + (bytes, offset, time, isOrderTimeNull, updater) -> { + float value = BytesUtils.bytesToFloat(bytes, offset); + updater.updateFloat(value, time, isOrderTimeNull); + }; + case DOUBLE -> + (bytes, offset, time, isOrderTimeNull, updater) -> { + double value = BytesUtils.bytesToDouble(bytes, offset); + updater.updateDouble(value, time, isOrderTimeNull); + }; + case TEXT, BLOB, OBJECT, STRING -> + (bytes, offset, time, isOrderTimeNull, updater) -> { + int length = BytesUtils.bytesToInt(bytes, offset); + Binary value = + new Binary(BytesUtils.subBytes(bytes, offset + Integer.BYTES, length)); + updater.updateBinary(value, time, isOrderTimeNull); + }; + case BOOLEAN -> + (bytes, offset, time, isOrderTimeNull, updater) -> + updater.updateBoolean( + BytesUtils.bytesToBool(bytes, offset), time, isOrderTimeNull); + case ROW, UNKNOWN, VECTOR -> + (bytes, offset, time, isOrderTimeNull, updater) -> { + throw new UnSupportedDataTypeException( + CalcMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()); + }; + }; + + public static final TypeService> + PRIMITIVE_TYPE_VALUE_EXTRACTOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, INT64, TIMESTAMP, FLOAT, DOUBLE -> TsPrimitiveType::getValue; + case DATE -> primitiveType -> DateUtils.parseIntToLocalDate(primitiveType.getInt()); + case TEXT, BLOB, STRING -> + primitiveType -> { + final Binary binary = primitiveType.getBinary(); + return binary == null || binary.getValues() == null + ? Binary.EMPTY_VALUE + : binary; + }; + case OBJECT, ROW, UNKNOWN, VECTOR -> + primitiveType -> { + throw new UnSupportedDataTypeException( + CalcMessages.UNSUPPORTED_DATA_TYPE + primitiveType.getDataType()) + .setChecked(true); + }; + }; + + /** Reads, serializes, deserializes, and writes Mode values without TsPrimitiveType conversion. */ + public static final TypeService MODE_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + modeValueService( + type, + type::getBoolean, + BytesUtils::bytesToBool, + (value, bytes, offset) -> { + BytesUtils.boolToBytes((boolean) value, bytes, offset); + return Byte.BYTES; + }, + (columnBuilder, value) -> type.writeBoolean(columnBuilder, (boolean) value)); + case INT32, DATE -> + modeValueService( + type, + type::getInt, + BytesUtils::bytesToInt, + (value, bytes, offset) -> { + BytesUtils.intToBytes((int) value, bytes, offset); + return Integer.BYTES; + }, + (columnBuilder, value) -> type.writeInt(columnBuilder, (int) value)); + case INT64, TIMESTAMP -> + modeValueService( + type, + type::getLong, + (bytes, offset) -> BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset), + (value, bytes, offset) -> { + BytesUtils.longToBytes((long) value, bytes, offset); + return Long.BYTES; + }, + (columnBuilder, value) -> type.writeLong(columnBuilder, (long) value)); + case FLOAT -> + modeValueService( + type, + type::getFloat, + BytesUtils::bytesToFloat, + (value, bytes, offset) -> { + BytesUtils.floatToBytes((float) value, bytes, offset); + return Float.BYTES; + }, + (columnBuilder, value) -> type.writeFloat(columnBuilder, (float) value)); + case DOUBLE -> + modeValueService( + type, + type::getDouble, + BytesUtils::bytesToDouble, + (value, bytes, offset) -> { + BytesUtils.doubleToBytes((double) value, bytes, offset); + return Double.BYTES; + }, + (columnBuilder, value) -> type.writeDouble(columnBuilder, (double) value)); + case TEXT, STRING, BLOB -> + modeValueService( + type, + type::getBinary, + TypeServices::deserializeModeBinary, + TypeServices::serializeModeBinary, + (columnBuilder, value) -> type.writeBinary(columnBuilder, (Binary) value)); + case OBJECT, ROW, UNKNOWN, VECTOR -> unsupportedModeValueService(type); + }; + + // The caller supplies the exception so shared conversion keeps each aggregation API's contract. + public static final TypeService + NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE, INT64, TIMESTAMP, FLOAT, DOUBLE -> ignored -> type::getDouble; + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + exceptionSupplier -> + (column, position) -> { + throw exceptionSupplier.get(); + }; + }; + + /** Numeric inputs supported by SUM, AVG and percentile aggregations. */ + public static final TypeService + AGGREGATION_NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, INT64, FLOAT, DOUBLE -> ignored -> type::getDouble; + case BOOLEAN, DATE, TIMESTAMP, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + exceptionSupplier -> + (column, position) -> { + throw exceptionSupplier.get(); + }; + }; + + /** Numeric inputs supported by percentile aggregations, including TIMESTAMP. */ + public static final TypeService + PERCENTILE_NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, INT64, TIMESTAMP, FLOAT, DOUBLE -> ignored -> type::getDouble; + case BOOLEAN, DATE, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + exceptionSupplier -> + (column, position) -> { + throw exceptionSupplier.get(); + }; + }; + + /** Writes a numeric percentile result in the input column's native type. */ + public static final TypeService NUMERIC_RESULT_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> (builder, value) -> type.writeInt(builder, (int) value); + case INT64, TIMESTAMP -> (builder, value) -> type.writeLong(builder, (long) value); + case FLOAT -> (builder, value) -> type.writeFloat(builder, (float) value); + case DOUBLE -> (builder, value) -> type.writeDouble(builder, value); + case BOOLEAN, DATE, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + (builder, value) -> { + throw new UnSupportedDataTypeException( + CalcMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()); + }; + }; + + /** Updates a reusable primitive result with a column value when it is more extreme. */ + public static final TypeService MIN_COLUMN_VALUE_UPDATER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (result, column, position, initialized) -> { + int value = type.getInt(column, position); + if (!initialized || value < result.getInt()) { + result.setInt(value); + return true; + } + return false; + }; + case INT64, TIMESTAMP -> + (result, column, position, initialized) -> { + long value = type.getLong(column, position); + if (!initialized || value < result.getLong()) { + result.setLong(value); + return true; + } + return false; + }; + case FLOAT -> + (result, column, position, initialized) -> { + float value = type.getFloat(column, position); + if (!initialized || value < result.getFloat()) { + result.setFloat(value); + return true; + } + return false; + }; + case DOUBLE -> + (result, column, position, initialized) -> { + double value = type.getDouble(column, position); + if (!initialized || value < result.getDouble()) { + result.setDouble(value); + return true; + } + return false; + }; + case BOOLEAN -> + (result, column, position, initialized) -> { + boolean value = type.getBoolean(column, position); + if (!initialized || !value) { + result.setBoolean(value); + return true; + } + return false; + }; + case TEXT, STRING, BLOB -> + (result, column, position, initialized) -> { + Binary value = type.getBinary(column, position); + if (!initialized || value.compareTo(result.getBinary()) < 0) { + result.setBinary(value); + return true; + } + return false; + }; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (result, column, position, initialized) -> { + throw new UnSupportedDataTypeException( + CalcMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()); + }; + }; + + /** Updates a reusable primitive result with a column value when it is more extreme. */ + public static final TypeService MAX_COLUMN_VALUE_UPDATER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (result, column, position, initialized) -> { + int value = type.getInt(column, position); + if (!initialized || value > result.getInt()) { + result.setInt(value); + return true; + } + return false; + }; + case INT64, TIMESTAMP -> + (result, column, position, initialized) -> { + long value = type.getLong(column, position); + if (!initialized || value > result.getLong()) { + result.setLong(value); + return true; + } + return false; + }; + case FLOAT -> + (result, column, position, initialized) -> { + float value = type.getFloat(column, position); + if (!initialized || value > result.getFloat()) { + result.setFloat(value); + return true; + } + return false; + }; + case DOUBLE -> + (result, column, position, initialized) -> { + double value = type.getDouble(column, position); + if (!initialized || value > result.getDouble()) { + result.setDouble(value); + return true; + } + return false; + }; + case BOOLEAN -> + (result, column, position, initialized) -> { + boolean value = type.getBoolean(column, position); + if (!initialized || value) { + result.setBoolean(value); + return true; + } + return false; + }; + case TEXT, STRING, BLOB -> + (result, column, position, initialized) -> { + Binary value = type.getBinary(column, position); + if (!initialized || value.compareTo(result.getBinary()) > 0) { + result.setBinary(value); + return true; + } + return false; + }; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (result, column, position, initialized) -> { + throw new UnSupportedDataTypeException( + CalcMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()); + }; + }; + + /** Updates a reusable primitive result with the value having the greatest absolute magnitude. */ + public static final TypeService EXTREME_COLUMN_VALUE_UPDATER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (result, column, position, initialized) -> { + int value = type.getInt(column, position); + int candidate = result.getInt(); + if (!initialized || compareExtreme(value, candidate) > 0) { + result.setInt(value); + return true; + } + return false; + }; + case INT64 -> + (result, column, position, initialized) -> { + long value = type.getLong(column, position); + long candidate = result.getLong(); + if (!initialized || compareExtreme(value, candidate) > 0) { + result.setLong(value); + return true; + } + return false; + }; + case FLOAT -> + (result, column, position, initialized) -> { + float value = type.getFloat(column, position); + float candidate = result.getFloat(); + float absValue = Math.abs(value); + float absCandidate = Math.abs(candidate); + if (!initialized + || absValue > absCandidate + || absValue == absCandidate && value > candidate) { + result.setFloat(value); + return true; + } + return false; + }; + case DOUBLE -> + (result, column, position, initialized) -> { + double value = type.getDouble(column, position); + double candidate = result.getDouble(); + double absValue = Math.abs(value); + double absCandidate = Math.abs(candidate); + if (!initialized + || absValue > absCandidate + || absValue == absCandidate && value > candidate) { + result.setDouble(value); + return true; + } + return false; + }; + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (result, column, position, initialized) -> { + throw new UnSupportedDataTypeException( + CalcMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()); + }; + }; + + /** Adds a column value to an HLL using the column's native primitive accessor. */ + public static final TypeService HYPER_LOG_LOG_COLUMN_ADDER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (hll, column, position) -> hll.add(column.getBoolean(position)); + case INT32, DATE -> (hll, column, position) -> hll.add(column.getInt(position)); + case INT64, TIMESTAMP -> (hll, column, position) -> hll.add(column.getLong(position)); + case FLOAT -> (hll, column, position) -> hll.add(column.getFloat(position)); + case DOUBLE -> (hll, column, position) -> hll.add(column.getDouble(position)); + case TEXT, BLOB, STRING, OBJECT -> + (hll, column, position) -> hll.add(column.getBinary(position)); + case ROW, UNKNOWN, VECTOR -> + (hll, column, position) -> { + throw new UnSupportedDataTypeException( + CalcMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()) + .setChecked(true); + }; + }; + + // RANGE frame offsets must retain each primitive type's native overflow and precision rules. + public static final TypeService RANGE_FRAME_COMPARATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (column, + partition, + currentIndex, + recentIndex, + channel, + offsetOperation, + comparison) -> { + int current = column.getInt(currentIndex); + int offset = partition.getInt(channel, currentIndex); + int boundary = offsetOperation.apply(current, offset); + return comparison.compare(column.getInt(recentIndex), boundary); + }; + case INT64, TIMESTAMP -> + (column, + partition, + currentIndex, + recentIndex, + channel, + offsetOperation, + comparison) -> { + long current = column.getLong(currentIndex); + long offset = partition.getLong(channel, currentIndex); + long boundary = offsetOperation.apply(current, offset); + return comparison.compare(column.getLong(recentIndex), boundary); + }; + case FLOAT -> + (column, + partition, + currentIndex, + recentIndex, + channel, + offsetOperation, + comparison) -> { + float current = column.getFloat(currentIndex); + float offset = partition.getFloat(channel, currentIndex); + float boundary = offsetOperation.apply(current, offset); + return comparison.compare(column.getFloat(recentIndex), boundary); + }; + case DOUBLE -> + (column, + partition, + currentIndex, + recentIndex, + channel, + offsetOperation, + comparison) -> { + double current = column.getDouble(currentIndex); + double offset = partition.getDouble(channel, currentIndex); + double boundary = offsetOperation.apply(current, offset); + return comparison.compare(column.getDouble(recentIndex), boundary); + }; + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + (column, + partition, + currentIndex, + recentIndex, + channel, + offsetOperation, + comparison) -> { + throw new UnSupportedDataTypeException(CalcMessages.UNSUPPORTED_DATA_TYPE + type); + }; + }; + + public static final TypeService> + DEFAULT_ENCODING_BY_TYPE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> DefaultEncodingProvider::getDefaultBooleanEncoding; + case INT32, DATE -> DefaultEncodingProvider::getDefaultInt32Encoding; + case INT64, TIMESTAMP -> DefaultEncodingProvider::getDefaultInt64Encoding; + case FLOAT -> DefaultEncodingProvider::getDefaultFloatEncoding; + case DOUBLE -> DefaultEncodingProvider::getDefaultDoubleEncoding; + case STRING, BLOB, OBJECT, TEXT -> DefaultEncodingProvider::getDefaultTextEncoding; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(CalcMessages.UNKNOWN_DATATYPE + type) + .setChecked(true); + }; + + public static final TypeService DEFAULT_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (partition, channel, index, builder) -> + builder.writeInt(partition.getInt(channel, index)); + case INT64, TIMESTAMP -> + (partition, channel, index, builder) -> + builder.writeLong(partition.getLong(channel, index)); + case FLOAT -> + (partition, channel, index, builder) -> + builder.writeFloat(partition.getFloat(channel, index)); + case DOUBLE -> + (partition, channel, index, builder) -> + builder.writeDouble(partition.getDouble(channel, index)); + case BOOLEAN -> + (partition, channel, index, builder) -> + builder.writeBoolean(partition.getBoolean(channel, index)); + case TEXT, STRING, BLOB, OBJECT -> + (partition, channel, index, builder) -> + builder.writeBinary(partition.getBinary(channel, index)); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + CalcMessages.UNSUPPORTED_DEFAULT_VALUE_DATA_TYPE_IN_LAG + type) + .setChecked(true); + }; + + public static final TypeService INTERMEDIATE_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (accumulator, isX, index, bytes, offset) -> + (isX ? accumulator.getXIntValues() : accumulator.getYIntValues()) + .toBytes(index, bytes, offset); + case INT64, TIMESTAMP -> + (accumulator, isX, index, bytes, offset) -> + (isX ? accumulator.getXLongValues() : accumulator.getYLongValues()) + .toBytes(index, bytes, offset); + case FLOAT -> + (accumulator, isX, index, bytes, offset) -> + (isX ? accumulator.getXFloatValues() : accumulator.getYFloatValues()) + .toBytes(index, bytes, offset); + case DOUBLE -> + (accumulator, isX, index, bytes, offset) -> + (isX ? accumulator.getXDoubleValues() : accumulator.getYDoubleValues()) + .toBytes(index, bytes, offset); + case TEXT, STRING, BLOB, OBJECT -> + (accumulator, isX, index, bytes, offset) -> + (isX ? accumulator.getXBinaryValues() : accumulator.getYBinaryValues()) + .toBytes(index, bytes, offset); + case BOOLEAN -> + (accumulator, isX, index, bytes, offset) -> + (isX ? accumulator.getXBooleanValues() : accumulator.getYBooleanValues()) + .toBytes(index, bytes, offset); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(CalcMessages.UNKNOWN_DATATYPE + type) + .setChecked(true); + }; + + public static final TypeService + INTERMEDIATE_VALUE_INITIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (accumulator, isX) -> { + if (isX) { + accumulator.setXIntValues(new IntBigArray()); + } else { + accumulator.setYIntValues(new IntBigArray()); + } + }; + case INT64, TIMESTAMP -> + (accumulator, isX) -> { + if (isX) { + accumulator.setXLongValues(new LongBigArray()); + } else { + accumulator.setYLongValues(new LongBigArray()); + } + }; + case FLOAT -> + (accumulator, isX) -> { + if (isX) { + accumulator.setXFloatValues(new FloatBigArray()); + } else { + accumulator.setYFloatValues(new FloatBigArray()); + } + }; + case DOUBLE -> + (accumulator, isX) -> { + if (isX) { + accumulator.setXDoubleValues(new DoubleBigArray()); + } else { + accumulator.setYDoubleValues(new DoubleBigArray()); + } + }; + case TEXT, STRING, BLOB, OBJECT -> + (accumulator, isX) -> { + if (isX) { + accumulator.setXBinaryValues(new BinaryBigArray()); + } else { + accumulator.setYBinaryValues(new BinaryBigArray()); + } + }; + case BOOLEAN -> + (accumulator, isX) -> { + if (isX) { + accumulator.setXBooleanValues(new BooleanBigArray()); + } else { + accumulator.setYBooleanValues(new BooleanBigArray()); + } + }; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(CalcMessages.UNKNOWN_DATATYPE + type) + .setChecked(true); + }; + + static { + LINEAR_FILL_SERVICE.check(); + PREVIOUS_FILL_SERVICE.check(); + NEXT_FILL_SERVICE.check(); + VALUE_FILL_SERVICE.check(); + JOIN_KEY_COMPARATOR_SERVICE.check(); + GREATEST_COLUMN_TRANSFORMER_SERVICE.check(); + LEAST_COLUMN_TRANSFORMER_SERVICE.check(); + IN_COLUMN_VALUE_MATCHER_SERVICE.check(); + IN_MULTI_COLUMN_TRANSFORMER_SERVICE.check(); + UPDATE_LAST_ROW_SERVICE.check(); + PRIMITIVE_COLUMN_VALUE_SETTER_SERVICE.check(); + STATISTICS_VALUE_SETTER_SERVICE.check(); + MIN_STATISTICS_VALUE_UPDATER_SERVICE.check(); + MAX_STATISTICS_VALUE_UPDATER_SERVICE.check(); + EXTREME_STATISTICS_VALUE_UPDATER_SERVICE.check(); + MAX_MIN_BY_INPUT_SERVICE.check(); + MAX_MIN_BY_INTERMEDIATE_INPUT_SERVICE.check(); + LAST_VALUE_DESERIALIZER_SERVICE.check(); + MERGE_SORT_COMPARATOR_SERVICE.check(); + MEMORY_USAGE_OF_ONE_MERGE_SORT_KEY_SERVICE.check(); + MEMORY_USAGE_OF_ONE_SERIALIZABLE_ROW_FIELD_SERVICE.check(); + PRIMITIVE_TYPE_VALUE_EXTRACTOR_SERVICE.check(); + MODE_VALUE_SERVICE.check(); + NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE.check(); + AGGREGATION_NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE.check(); + PERCENTILE_NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE.check(); + NUMERIC_RESULT_WRITER_SERVICE.check(); + MIN_COLUMN_VALUE_UPDATER_SERVICE.check(); + MAX_COLUMN_VALUE_UPDATER_SERVICE.check(); + EXTREME_COLUMN_VALUE_UPDATER_SERVICE.check(); + HYPER_LOG_LOG_COLUMN_ADDER_SERVICE.check(); + RANGE_FRAME_COMPARATOR_SERVICE.check(); + DEFAULT_ENCODING_BY_TYPE_SERVICE.check(); + DEFAULT_VALUE_WRITER_SERVICE.check(); + INTERMEDIATE_VALUE_WRITER_SERVICE.check(); + INTERMEDIATE_VALUE_INITIALIZER_SERVICE.check(); + CAST_INPUT_SERVICE.check(); + CAST_VALUE_SERVICE.check(); + CAST_OBJECT_VALUE_SERVICE.check(); + CAST_OBJECT_INPUT_SERVICE.check(); + FORMAT_VALUE_CONVERTER_SERVICE.check(); + GROUPED_APPROX_MOST_FREQUENT_ACCUMULATOR_SERVICE.check(); + APPROX_MOST_FREQUENT_ACCUMULATOR_SERVICE.check(); + APPROX_PERCENTILE_INPUT_SERVICE.check(); + GROUPED_APPROX_PERCENTILE_INPUT_SERVICE.check(); + GROUPED_VALUE_SERVICE.check(); + GROUPED_INPUT_READER_SERVICE.check(); + GROUPED_EXTREME_VALUE_SERVICE.check(); + GROUPED_EXTREME_INPUT_READER_SERVICE.check(); + GROUPED_TIME_INPUT_READER_SERVICE.check(); + GROUPED_BY_INPUT_READER_SERVICE.check(); + GROUPED_VALUE_SERIALIZER_SERVICE.check(); + GROUPED_VALUE_SETTER_SERVICE.check(); + GROUPED_TIME_VALUE_DESERIALIZER_SERVICE.check(); + GROUPED_BY_VALUE_DESERIALIZER_SERVICE.check(); + GROUPED_MAX_MIN_BY_INPUT_SERVICE.check(); + GROUPED_MAX_MIN_BY_INTERMEDIATE_INPUT_SERVICE.check(); + } + + private TypeServices() { + // util class doesn't need constructor + } + + private static IllegalArgumentException unsupportedDataType(final Type type) { + return new IllegalArgumentException(CalcMessages.UNKNOWN_DATATYPE + type.getTypeEnum()); + } + + private static UnsupportedOperationException unsupportedGreatestLeastDataType(final Type type) { + return new UnsupportedOperationException( + CalcMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()); + } + + private static UnsupportedOperationException unsupportedCastTarget(final Type type) { + return new UnsupportedOperationException( + String.format("Unsupported target dataType: %s", type.getTypeEnum())); + } + + private static UnsupportedOperationException unsupportedCastSource(final Type type) { + return new UnsupportedOperationException( + String.format("Unsupported source dataType: %s", type.getTypeEnum())); + } + + private static UnSupportedDataTypeException unsupportedApproxMostFrequentDataType( + final Type type) { + return new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_APPROX_COUNT_DISTINCT_AGGREGATION, + type.getTypeEnum())) + .setChecked(true); + } + + private static GroupedValueService unsupportedGroupedValueService(final Type type) { + return new GroupedValueService() { + @Override + public void initialize(GroupedValueAccessor accessor) { + throw accessor.unsupportedException(); + } + + @Override + public void ensureCapacity(GroupedValueAccessor accessor, long groupCount) { + throw accessor.unsupportedException(); + } + + @Override + public long sizeOf(GroupedValueAccessor accessor) { + throw accessor.unsupportedException(); + } + + @Override + public void reset(GroupedValueAccessor accessor) { + throw accessor.unsupportedException(); + } + + @Override + public void write(GroupedValueAccessor accessor, int groupId, ColumnBuilder columnBuilder) { + throw accessor.unsupportedException(); + } + }; + } + + private static GroupedInputReader unsupportedGroupedInputReader(final Type type) { + return new GroupedInputReader() { + @Override + public void addInput( + int[] groupIds, Column[] arguments, AggregationMask mask, GroupedInputConsumer consumer) { + throw unsupportedDataType(type); + } + + @Override + public void addIntermediate(int[] groupIds, Column argument, GroupedInputConsumer consumer) { + throw unsupportedDataType(type); + } + }; + } + + private static GroupedValueSerializer unsupportedGroupedValueSerializer(final Type type) { + return new GroupedValueSerializer() { + @Override + public int calcTypeSize(GroupedValueAccessor accessor, long index) { + throw unsupportedDataType(type); + } + + @Override + public void serialize(GroupedValueAccessor accessor, long index, byte[] bytes, int offset) { + throw unsupportedDataType(type); + } + }; + } + + private static UnSupportedDataTypeException unsupportedMaxMinByDataType(final Type type) { + return new UnSupportedDataTypeException( + String.format( + CalcMessages.UNSUPPORTED_DATA_TYPE_IN_MAX_BY_MIN_BY_AGGREGATION, type.getTypeEnum())); + } + + private static ModeValueService modeValueService( + Type type, + ModeValueGetter getter, + ModeValueDeserializer deserializer, + ModeValueSerializer serializer, + ModeValueWriter writer) { + return new ModeValueService() { + @Override + public Object getValue(Column column, int position) { + return getter.get(column, position); + } + + @Override + public Object deserialize(byte[] bytes, int offset) { + return deserializer.deserialize(bytes, offset); + } + + @Override + public int calcTypeSize(Object value) { + return type.calcTypeSize(value); + } + + @Override + public int serialize(Object value, byte[] bytes, int offset) { + return serializer.serialize(value, bytes, offset); + } + + @Override + public void write(ColumnBuilder columnBuilder, Object value) { + writer.write(columnBuilder, value); + } + }; + } + + private static ModeValueService unsupportedModeValueService(final Type type) { + return new ModeValueService() { + @Override + public Object getValue(Column column, int position) { + throw unsupportedModeValueDataType(type); + } + + @Override + public Object deserialize(byte[] bytes, int offset) { + throw unsupportedModeValueDataType(type); + } + + @Override + public int calcTypeSize(Object value) { + throw unsupportedModeValueDataType(type); + } + + @Override + public int serialize(Object value, byte[] bytes, int offset) { + throw unsupportedModeValueDataType(type); + } + + @Override + public void write(ColumnBuilder columnBuilder, Object value) { + throw unsupportedModeValueDataType(type); + } + }; + } + + private static UnSupportedDataTypeException unsupportedModeValueDataType(final Type type) { + return new UnSupportedDataTypeException(CalcMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()) + .setChecked(true); + } + + private static int serializeModeBinary(Object value, byte[] bytes, int offset) { + Binary binary = (Binary) value; + BytesUtils.intToBytes(binary.getLength(), bytes, offset); + System.arraycopy(binary.getValues(), 0, bytes, offset + Integer.BYTES, binary.getLength()); + return Integer.BYTES + binary.getLength(); + } + + private static Binary deserializeModeBinary(byte[] bytes, int offset) { + int length = BytesUtils.bytesToInt(bytes, offset); + return new Binary(BytesUtils.subBytes(bytes, offset + Integer.BYTES, length)); + } + + private static Binary toBinary(Object value) { + return value instanceof Binary + ? (Binary) value + : new Binary(String.valueOf(value), java.nio.charset.StandardCharsets.UTF_8); + } + + private static void forEachSelectedPosition( + Column column, AggregationMask mask, IntConsumer positionConsumer) { + int positionCount = mask.getSelectedPositionCount(); + if (mask.isSelectAll()) { + for (int position = 0; position < positionCount; position++) { + if (!column.isNull(position)) { + positionConsumer.accept(position); + } + } + } else { + int[] selectedPositions = mask.getSelectedPositions(); + for (int i = 0; i < positionCount; i++) { + int position = selectedPositions[i]; + if (!column.isNull(position)) { + positionConsumer.accept(position); + } + } + } + } + + private static void readMaxMinByXFromBytes( + byte[] bytes, int offset, Type xType, ColumnBuilder columnBuilder) { + if (BytesUtils.bytesToBool(bytes, offset)) { + columnBuilder.appendNull(); + } else { + xType.write(columnBuilder, bytes, offset + 1); + } + } + + private static int compareExtreme(int left, int right) { + int absComparison = Long.compare(Math.abs((long) left), Math.abs((long) right)); + return absComparison == 0 ? Integer.compare(left, right) : absComparison; + } + + private static int compareExtreme(long left, long right) { + int absComparison = compareAbs(left, right); + return absComparison == 0 ? Long.compare(left, right) : absComparison; + } + + private static int compareAbs(long left, long right) { + if (left == Long.MIN_VALUE) { + return right == Long.MIN_VALUE ? 0 : 1; + } + if (right == Long.MIN_VALUE) { + return -1; + } + return Long.compare(Math.abs(left), Math.abs(right)); + } + + public interface DefaultEncodingProvider { + TSEncoding getDefaultBooleanEncoding(); + + TSEncoding getDefaultInt32Encoding(); + + TSEncoding getDefaultInt64Encoding(); + + TSEncoding getDefaultFloatEncoding(); + + TSEncoding getDefaultDoubleEncoding(); + + TSEncoding getDefaultTextEncoding(); + } + + @FunctionalInterface + public interface ColumnRowFunction { + Column apply(Column column, int rowIndex); + } + + @FunctionalInterface + public interface InColumnValueMatcherFactory { + InColumnValueMatcher create(Set values); + } + + @FunctionalInterface + public interface InColumnValueMatcher { + boolean matches(Column column, int position); + } + + @FunctionalInterface + public interface InMultiColumnTransformerFactory { + InMultiColumnTransformer create( + List valueColumnTransformers, List values); + } + + @FunctionalInterface + public interface CastInputService { + void cast( + Column column, + ColumnBuilder columnBuilder, + int position, + CastValueService castValueService, + ZoneId zoneId); + } + + public interface CastValueService { + void castInt(ColumnBuilder columnBuilder, int value); + + void castDate(ColumnBuilder columnBuilder, int value, ZoneId zoneId); + + void castTimestamp(ColumnBuilder columnBuilder, long value, ZoneId zoneId); + + void castLong(ColumnBuilder columnBuilder, long value); + + void castFloat(ColumnBuilder columnBuilder, float value); + + void castDouble(ColumnBuilder columnBuilder, double value); + + void castBoolean(ColumnBuilder columnBuilder, boolean value); + + void castString(ColumnBuilder columnBuilder, Binary value, ZoneId zoneId); + + void castBlob(ColumnBuilder columnBuilder, Binary value, ZoneId zoneId); + + void castObject(ColumnBuilder columnBuilder, Binary value); + } + + public interface CastObjectValueService { + Object castInt(int value); + + Object castDate(int value, ZoneId zoneId); + + Object castTimestamp(long value, ZoneId zoneId); + + Object castLong(long value); + + Object castFloat(float value); + + Object castDouble(double value); + + Object castBoolean(boolean value); + + Object castBinary(Binary value, ZoneId zoneId); + } + + @FunctionalInterface + public interface CastObjectInputService { + Object cast(Object value, CastObjectValueService castValueService, ZoneId zoneId); + } + + @FunctionalInterface + public interface FormatValueConverter { + Object convert(Column column, int position, ZoneId zoneId); + } + + @FunctionalInterface + public interface PrimitiveColumnValueSetter { + void set(TsPrimitiveType target, Column column, int position); + } + + @FunctionalInterface + public interface StatisticsValueSetter { + void set( + TsPrimitiveType target, + Object value, + Supplier exceptionSupplier); + } + + @FunctionalInterface + public interface StatisticsValueUpdater { + boolean update( + TsPrimitiveType result, + Object value, + boolean initialized, + Supplier exceptionSupplier); + } + + @FunctionalInterface + public interface TriConsumer { + void accept(A first, B second, C third); + } + + public interface ModeValueService { + Object getValue(Column column, int position); + + Object deserialize(byte[] bytes, int offset); + + int calcTypeSize(Object value); + + int serialize(Object value, byte[] bytes, int offset); + + void write(ColumnBuilder columnBuilder, Object value); + } + + @FunctionalInterface + private interface ModeValueGetter { + Object get(Column column, int position); + } + + @FunctionalInterface + private interface ModeValueDeserializer { + Object deserialize(byte[] bytes, int offset); + } + + @FunctionalInterface + private interface ModeValueSerializer { + int serialize(Object value, byte[] bytes, int offset); + } + + @FunctionalInterface + private interface ModeValueWriter { + void write(ColumnBuilder columnBuilder, Object value); + } + + public interface MaxMinByValueUpdater { + void updateInt(int value, Column xColumn, int xIndex); + + void updateLong(long value, Column xColumn, int xIndex); + + void updateFloat(float value, Column xColumn, int xIndex); + + void updateDouble(double value, Column xColumn, int xIndex); + + void updateBinary(Binary value, Column xColumn, int xIndex); + + void updateBoolean(boolean value, Column xColumn, int xIndex); + } + + @FunctionalInterface + public interface MaxMinByInput { + void add(Column[] arguments, AggregationMask mask, MaxMinByValueUpdater updater); + } + + @FunctionalInterface + public interface MaxMinByIntermediateInput { + void update( + byte[] bytes, Type xType, ColumnBuilder columnBuilder, MaxMinByValueUpdater updater); + } + + @FunctionalInterface + public interface LastValueDeserializer { + void deserialize( + byte[] bytes, int offset, long time, boolean isOrderTimeNull, LastValueUpdater updater); + } + + public interface LastValueUpdater { + void updateInt(int value, long time, boolean isOrderTimeNull); + + void updateLong(long value, long time, boolean isOrderTimeNull); + + void updateFloat(float value, long time, boolean isOrderTimeNull); + + void updateDouble(double value, long time, boolean isOrderTimeNull); + + void updateBinary(Binary value, long time, boolean isOrderTimeNull); + + void updateBoolean(boolean value, long time, boolean isOrderTimeNull); + } + + @FunctionalInterface + public interface ColumnToDoubleConverter { + double convert(Column column, int position); + } + + @FunctionalInterface + public interface BooleanFunction { + // Keep primitive dispatchers unboxed when selecting ascending/descending strategies. + T apply(boolean value); + } + + @FunctionalInterface + public interface ColumnToDoubleConverterFactory { + ColumnToDoubleConverter create(Supplier exceptionSupplier); + } + + @FunctionalInterface + public interface NumericResultWriter { + void write(ColumnBuilder builder, double value); + } + + @FunctionalInterface + public interface ColumnValueUpdater { + boolean update(TsPrimitiveType result, Column column, int position, boolean initialized); + } + + @FunctionalInterface + public interface HyperLogLogColumnAdder { + void add(HyperLogLog hll, Column column, int position); + } + + @FunctionalInterface + public interface RangeFrameComparator { + boolean compare( + ColumnList column, + Partition partition, + int currentIndex, + int recentIndex, + int channel, + RangeFrameOffsetOperation offsetOperation, + RangeFrameComparison comparison); + } + + public enum RangeFrameOffsetOperation { + ADD { + @Override + int apply(int value, int offset) { + return value + offset; + } + + @Override + long apply(long value, long offset) { + return value + offset; + } + + @Override + float apply(float value, float offset) { + return value + offset; + } + + @Override + double apply(double value, double offset) { + return value + offset; + } + }, + SUBTRACT { + @Override + int apply(int value, int offset) { + return value - offset; + } + + @Override + long apply(long value, long offset) { + return value - offset; + } + + @Override + float apply(float value, float offset) { + return value - offset; + } + + @Override + double apply(double value, double offset) { + return value - offset; + } + }; + + abstract int apply(int value, int offset); + + abstract long apply(long value, long offset); + + abstract float apply(float value, float offset); + + abstract double apply(double value, double offset); + } + + public enum RangeFrameComparison { + GREATER_THAN_OR_EQUAL { + @Override + boolean compare(int left, int right) { + return left >= right; + } + + @Override + boolean compare(long left, long right) { + return left >= right; + } + + @Override + boolean compare(float left, float right) { + return left >= right; + } + + @Override + boolean compare(double left, double right) { + return left >= right; + } + }, + GREATER_THAN { + @Override + boolean compare(int left, int right) { + return left > right; + } + + @Override + boolean compare(long left, long right) { + return left > right; + } + + @Override + boolean compare(float left, float right) { + return left > right; + } + + @Override + boolean compare(double left, double right) { + return left > right; + } + }, + LESS_THAN_OR_EQUAL { + @Override + boolean compare(int left, int right) { + return left <= right; + } + + @Override + boolean compare(long left, long right) { + return left <= right; + } + + @Override + boolean compare(float left, float right) { + return left <= right; + } + + @Override + boolean compare(double left, double right) { + return left <= right; + } + }, + LESS_THAN { + @Override + boolean compare(int left, int right) { + return left < right; + } + + @Override + boolean compare(long left, long right) { + return left < right; + } + + @Override + boolean compare(float left, float right) { + return left < right; + } + + @Override + boolean compare(double left, double right) { + return left < right; + } + }; + + abstract boolean compare(int left, int right); + + abstract boolean compare(long left, long right); + + abstract boolean compare(float left, float right); + + abstract boolean compare(double left, double right); + } + + @FunctionalInterface + public interface DefaultValueWriter { + void write(Partition partition, int channel, int index, ColumnBuilder builder); + } + + @FunctionalInterface + public interface IntermediateValueWriter { + void write( + GroupedMaxMinByBaseAccumulator accumulator, + boolean isX, + long index, + byte[] bytes, + int offset); + } + + @FunctionalInterface + public interface IntermediateValueInitializer { + void initialize(GroupedMaxMinByBaseAccumulator accumulator, boolean isX); + } + + public interface GroupedValueAccessor { + RuntimeException unsupportedException(); + + void initializeIntValues(); + + void initializeLongValues(); + + void initializeFloatValues(); + + void initializeDoubleValues(); + + void initializeBinaryValues(); + + void initializeBooleanValues(); + + IntBigArray getIntValues(); + + LongBigArray getLongValues(); + + FloatBigArray getFloatValues(); + + DoubleBigArray getDoubleValues(); + + BinaryBigArray getBinaryValues(); + + BooleanBigArray getBooleanValues(); + } + + public interface GroupedValueService { + void initialize(GroupedValueAccessor accessor); + + void ensureCapacity(GroupedValueAccessor accessor, long groupCount); + + long sizeOf(GroupedValueAccessor accessor); + + void reset(GroupedValueAccessor accessor); + + void write(GroupedValueAccessor accessor, int groupId, ColumnBuilder columnBuilder); + } + + public interface GroupedInputConsumer { + void updateInt(int groupId, int value); + + void updateLong(int groupId, long value); + + void updateFloat(int groupId, float value); + + void updateDouble(int groupId, double value); + + void updateBinary(int groupId, Binary value); + + void updateBoolean(int groupId, boolean value); + } + + public interface GroupedInputReader { + void addInput( + int[] groupIds, Column[] arguments, AggregationMask mask, GroupedInputConsumer consumer); + + void addIntermediate(int[] groupIds, Column argument, GroupedInputConsumer consumer); + } + + public interface GroupedTimeInputReader { + void addInput( + int[] groupIds, + Column[] arguments, + AggregationMask mask, + GroupedTimeValueConsumer consumer); + } + + public interface GroupedTimeValueConsumer { + void updateInt(int groupId, int value, boolean timeNull, long time); + + void updateLong(int groupId, long value, boolean timeNull, long time); + + void updateFloat(int groupId, float value, boolean timeNull, long time); + + void updateDouble(int groupId, double value, boolean timeNull, long time); + + void updateBinary(int groupId, Binary value, boolean timeNull, long time); + + void updateBoolean(int groupId, boolean value, boolean timeNull, long time); + } + + public interface GroupedByInputReader { + void addInput( + int[] groupIds, Column[] arguments, AggregationMask mask, GroupedByValueConsumer consumer); + } + + public interface GroupedByValueConsumer { + void updateInt(int groupId, boolean xNull, int value, boolean timeNull, long time); + + void updateLong(int groupId, boolean xNull, long value, boolean timeNull, long time); + + void updateFloat(int groupId, boolean xNull, float value, boolean timeNull, long time); + + void updateDouble(int groupId, boolean xNull, double value, boolean timeNull, long time); + + void updateBinary(int groupId, boolean xNull, Binary value, boolean timeNull, long time); + + void updateBoolean(int groupId, boolean xNull, boolean value, boolean timeNull, long time); + } + + public interface GroupedValueSerializer { + int calcTypeSize(GroupedValueAccessor accessor, long index); + + void serialize(GroupedValueAccessor accessor, long index, byte[] bytes, int offset); + } + + @FunctionalInterface + public interface GroupedValueSetter { + void set(GroupedValueAccessor accessor, long index, Column column, int position); + } + + @FunctionalInterface + public interface GroupedTimeValueDeserializer { + void deserialize( + byte[] bytes, + int offset, + int groupId, + long time, + boolean timeNull, + GroupedTimeValueConsumer consumer); + } + + @FunctionalInterface + public interface GroupedByValueDeserializer { + void deserialize( + byte[] bytes, + int offset, + int groupId, + boolean xNull, + long time, + boolean timeNull, + GroupedByValueConsumer consumer); + } + + public interface GroupedMaxMinByValueUpdater { + void updateInt(int groupId, int value, Column xColumn, int xIndex); + + void updateLong(int groupId, long value, Column xColumn, int xIndex); + + void updateFloat(int groupId, float value, Column xColumn, int xIndex); + + void updateDouble(int groupId, double value, Column xColumn, int xIndex); + + void updateBinary(int groupId, Binary value, Column xColumn, int xIndex); + + void updateBoolean(int groupId, boolean value, Column xColumn, int xIndex); + } + + @FunctionalInterface + public interface GroupedMaxMinByInput { + void add( + int[] groupIds, + Column[] arguments, + AggregationMask mask, + GroupedMaxMinByValueUpdater updater); + } + + @FunctionalInterface + public interface GroupedMaxMinByIntermediateInput { + void update( + int groupId, + byte[] bytes, + Type xType, + ColumnBuilder columnBuilder, + GroupedMaxMinByValueUpdater updater); + } + + @FunctionalInterface + public interface ValueFillFactory { + IFill create(Literal literal, ZoneId zoneId); + } +} diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/VarianceAccumulatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/VarianceAccumulatorTest.java new file mode 100644 index 000000000000..a11bad4f5e17 --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/VarianceAccumulatorTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation; + +import org.apache.iotdb.calc.execution.aggregation.VarianceAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedVarianceAccumulator; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.read.common.block.column.FloatColumn; +import org.apache.tsfile.read.common.block.column.IntColumn; +import org.apache.tsfile.read.common.block.column.LongColumn; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Optional; + +public class VarianceAccumulatorTest { + + @Test + public void testVarianceAccumulatorsReadAllNumericTypes() { + TSDataType[] dataTypes = { + TSDataType.INT32, TSDataType.INT64, TSDataType.FLOAT, TSDataType.DOUBLE + }; + Column[] valueColumns = { + new IntColumn(2, Optional.empty(), new int[] {1, 3}), + new LongColumn(2, Optional.empty(), new long[] {1, 3}), + new FloatColumn(2, Optional.empty(), new float[] {1, 3}), + new DoubleColumn(2, Optional.empty(), new double[] {1, 3}) + }; + for (int i = 0; i < dataTypes.length; i++) { + TSDataType dataType = dataTypes[i]; + Column valueColumn = valueColumns[i]; + + VarianceAccumulator treeAccumulator = + new VarianceAccumulator(dataType, VarianceAccumulator.VarianceType.VAR_POP); + treeAccumulator.addInput(new Column[] {valueColumn, valueColumn}, /* bitMap= */ null); + DoubleColumnBuilder treeResult = new DoubleColumnBuilder(null, 1); + treeAccumulator.outputFinal(treeResult); + Assert.assertEquals(1.0, treeResult.build().getDouble(0), 0.0); + + TableVarianceAccumulator tableAccumulator = + new TableVarianceAccumulator(dataType, VarianceAccumulator.VarianceType.VAR_POP); + tableAccumulator.addInput( + new Column[] {valueColumn}, + AggregationMask.createSelectAll(valueColumn.getPositionCount())); + DoubleColumnBuilder tableResult = new DoubleColumnBuilder(null, 1); + tableAccumulator.evaluateFinal(tableResult); + Assert.assertEquals(1.0, tableResult.build().getDouble(0), 0.0); + + GroupedVarianceAccumulator groupedAccumulator = + new GroupedVarianceAccumulator(dataType, VarianceAccumulator.VarianceType.VAR_POP); + groupedAccumulator.setGroupCount(1); + groupedAccumulator.addInput( + new int[] {0, 0}, + new Column[] {valueColumn}, + AggregationMask.createSelectAll(valueColumn.getPositionCount())); + DoubleColumnBuilder groupedResult = new DoubleColumnBuilder(null, 1); + groupedAccumulator.evaluateFinal(0, groupedResult); + Assert.assertEquals(1.0, groupedResult.build().getDouble(0), 0.0); + } + } + + @Test + public void testVarianceAccumulatorsStillRejectTemporalTypes() { + TSDataType[] dataTypes = {TSDataType.DATE, TSDataType.TIMESTAMP}; + Column[] valueColumns = { + new IntColumn(2, Optional.empty(), new int[] {1, 3}, TSDataType.DATE), + new LongColumn(2, Optional.empty(), new long[] {1, 3}) + }; + for (int i = 0; i < dataTypes.length; i++) { + TSDataType dataType = dataTypes[i]; + Column valueColumn = valueColumns[i]; + + VarianceAccumulator treeAccumulator = + new VarianceAccumulator(dataType, VarianceAccumulator.VarianceType.VAR_POP); + Assert.assertThrows( + UnSupportedDataTypeException.class, + () -> + treeAccumulator.addInput( + new Column[] {valueColumn, valueColumn}, /* bitMap= */ null)); + + TableVarianceAccumulator tableAccumulator = + new TableVarianceAccumulator(dataType, VarianceAccumulator.VarianceType.VAR_POP); + Assert.assertThrows( + UnSupportedDataTypeException.class, + () -> + tableAccumulator.addInput( + new Column[] {valueColumn}, + AggregationMask.createSelectAll(valueColumn.getPositionCount()))); + Assert.assertThrows( + UnSupportedDataTypeException.class, + () -> tableAccumulator.removeInput(new Column[] {valueColumn})); + + GroupedVarianceAccumulator groupedAccumulator = + new GroupedVarianceAccumulator(dataType, VarianceAccumulator.VarianceType.VAR_POP); + Assert.assertThrows( + UnSupportedDataTypeException.class, + () -> + groupedAccumulator.addInput( + new int[] {0, 0}, + new Column[] {valueColumn}, + AggregationMask.createSelectAll(valueColumn.getPositionCount()))); + } + } +} diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/utils/TypeServicesTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/utils/TypeServicesTest.java new file mode 100644 index 000000000000..ffa1f6965236 --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/utils/TypeServicesTest.java @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.calc.utils; + +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.AscIntTypeJoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.JoinKeyComparator; +import org.apache.iotdb.calc.execution.operator.process.join.merge.comparator.JoinKeyComparatorFactory; +import org.apache.iotdb.calc.execution.operator.process.window.partition.Partition; +import org.apache.iotdb.calc.execution.operator.process.window.utils.ColumnList; +import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.FloatColumn; +import org.apache.tsfile.read.common.block.column.IntColumn; +import org.apache.tsfile.read.common.block.column.LongColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class TypeServicesTest { + + @Test + public void testStatisticsValueSetterCreatesExceptionOnlyForUnsupportedType() { + AtomicInteger exceptionCount = new AtomicInteger(); + RuntimeException expected = new UnsupportedOperationException("unsupported statistics type"); + Supplier exceptionSupplier = + () -> { + exceptionCount.incrementAndGet(); + return expected; + }; + TsPrimitiveType result = Type.fromTsDataType(TSDataType.INT32).getTsPrimitiveType(); + + TypeServices.STATISTICS_VALUE_SETTER_SERVICE + .call(Type.fromTsDataType(TSDataType.INT32)) + .set(result, 42, exceptionSupplier); + assertEquals(42, result.getInt()); + assertEquals(0, exceptionCount.get()); + + assertSame( + expected, + Assert.assertThrows( + UnsupportedOperationException.class, + () -> + TypeServices.STATISTICS_VALUE_SETTER_SERVICE + .call(Type.fromTsDataType(TSDataType.VECTOR)) + .set(result, 42, exceptionSupplier))); + assertEquals(1, exceptionCount.get()); + } + + @Test + public void testNumericColumnToDoubleConversion() { + assertEquals( + 1.0, convert(TSDataType.INT32, new IntColumn(1, Optional.empty(), new int[] {1})), 0.0); + assertEquals( + 2.0, + convert( + TSDataType.DATE, new IntColumn(1, Optional.empty(), new int[] {2}, TSDataType.DATE)), + 0.0); + assertEquals( + 3.0, convert(TSDataType.INT64, new LongColumn(1, Optional.empty(), new long[] {3L})), 0.0); + assertEquals( + 4.0, + convert(TSDataType.TIMESTAMP, new LongColumn(1, Optional.empty(), new long[] {4L})), + 0.0); + assertEquals( + 5.5, + convert(TSDataType.FLOAT, new FloatColumn(1, Optional.empty(), new float[] {5.5F})), + 0.0); + assertEquals( + 6.5, + convert(TSDataType.DOUBLE, new DoubleColumn(1, Optional.empty(), new double[] {6.5D})), + 0.0); + } + + @Test + public void testStatisticsValueUpdatersCreateExceptionOnlyForUnsupportedType() { + for (org.apache.tsfile.read.common.type.service.TypeService + service : + List.of( + TypeServices.MAX_STATISTICS_VALUE_UPDATER_SERVICE, + TypeServices.MIN_STATISTICS_VALUE_UPDATER_SERVICE, + TypeServices.EXTREME_STATISTICS_VALUE_UPDATER_SERVICE)) { + AtomicInteger exceptionCount = new AtomicInteger(); + RuntimeException expected = new UnsupportedOperationException("unsupported statistics type"); + Supplier exceptionSupplier = + () -> { + exceptionCount.incrementAndGet(); + return expected; + }; + TsPrimitiveType result = Type.fromTsDataType(TSDataType.INT32).getTsPrimitiveType(); + TypeServices.StatisticsValueUpdater updater = + service.call(Type.fromTsDataType(TSDataType.INT32)); + assertTrue(updater.update(result, 42, false, exceptionSupplier)); + assertEquals(42, result.getInt()); + assertFalse(updater.update(result, 42, true, exceptionSupplier)); + assertEquals(0, exceptionCount.get()); + assertSame( + expected, + Assert.assertThrows( + UnsupportedOperationException.class, + () -> + service + .call(Type.fromTsDataType(TSDataType.VECTOR)) + .update(result, 42, true, exceptionSupplier))); + assertEquals(1, exceptionCount.get()); + } + } + + @Test + public void testNumericColumnToDoubleConversionUsesCallerException() { + final UnsupportedOperationException expected = new UnsupportedOperationException("expected"); + final TypeServices.ColumnToDoubleConverter converter = + TypeServices.NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(Type.fromTsDataType(TSDataType.BOOLEAN)) + .create(() -> expected); + + final UnsupportedOperationException actual = + Assert.assertThrows( + UnsupportedOperationException.class, + () -> converter.convert(new IntColumn(1, Optional.empty(), new int[] {1}), 0)); + + assertSame(expected, actual); + } + + @Test + public void testInColumnValueMatcherServiceUsesTypeStrategies() { + TypeServices.InColumnValueMatcher matcher = + TypeServices.IN_COLUMN_VALUE_MATCHER_SERVICE + .call(Type.fromTsDataType(TSDataType.INT32)) + .create(Collections.singleton("2")); + + Column input = new IntColumn(2, Optional.empty(), new int[] {1, 2}); + assertFalse(matcher.matches(input, 0)); + assertTrue(matcher.matches(input, 1)); + + Assert.assertThrows( + UnsupportedOperationException.class, + () -> + TypeServices.IN_COLUMN_VALUE_MATCHER_SERVICE + .call(Type.fromTsDataType(TSDataType.DATE)) + .create(Collections.emptySet())); + } + + @Test + public void testUnsupportedMergeSortComparatorFailsOnUse() { + Assert.assertThrows( + IllegalArgumentException.class, + () -> + TypeServices.MERGE_SORT_COMPARATOR_SERVICE + .call(Type.fromTsDataType(TSDataType.VECTOR)) + .apply(0)); + } + + @Test + public void testFillServicesUseTypeStrategies() { + assertNotNull( + TypeServices.LINEAR_FILL_SERVICE.call(Type.fromTsDataType(TSDataType.INT32)).get()); + assertNotNull( + TypeServices.PREVIOUS_FILL_SERVICE.call(Type.fromTsDataType(TSDataType.TEXT)).apply(null)); + assertNotNull( + TypeServices.NEXT_FILL_SERVICE.call(Type.fromTsDataType(TSDataType.DOUBLE)).apply(null)); + + Assert.assertThrows( + IllegalArgumentException.class, + () -> TypeServices.LINEAR_FILL_SERVICE.call(Type.fromTsDataType(TSDataType.UNKNOWN)).get()); + Assert.assertThrows( + IllegalArgumentException.class, + () -> + TypeServices.PREVIOUS_FILL_SERVICE + .call(Type.fromTsDataType(TSDataType.VECTOR)) + .apply(null)); + } + + @Test + public void testJoinKeyComparatorServiceUsesTypeStrategies() { + JoinKeyComparator comparator = + JoinKeyComparatorFactory.getComparator(Type.fromTsDataType(TSDataType.INT32), true); + assertSame(AscIntTypeJoinKeyComparator.getInstance(), comparator); + + Assert.assertThrows( + UnsupportedOperationException.class, + () -> JoinKeyComparatorFactory.getComparator(Type.fromTsDataType(TSDataType.VECTOR), true)); + } + + @Test + public void testGreatestLeastTransformerServicesUseTypeStrategies() { + List columnTransformers = Collections.emptyList(); + assertNotNull( + TypeServices.GREATEST_COLUMN_TRANSFORMER_SERVICE + .call(Type.fromTsDataType(TSDataType.INT32)) + .apply(columnTransformers)); + assertNotNull( + TypeServices.LEAST_COLUMN_TRANSFORMER_SERVICE + .call(Type.fromTsDataType(TSDataType.DOUBLE)) + .apply(columnTransformers)); + + Assert.assertThrows( + UnsupportedOperationException.class, + () -> + TypeServices.GREATEST_COLUMN_TRANSFORMER_SERVICE + .call(Type.fromTsDataType(TSDataType.VECTOR)) + .apply(columnTransformers)); + } + + @Test + public void testUpdateLastRowServiceUsesTypeColumnWriter() { + Column input = new IntColumn(2, Optional.empty(), new int[] {10, 20}, TSDataType.DATE); + Column output = + TypeServices.UPDATE_LAST_ROW_SERVICE + .call(Type.fromTsDataType(TSDataType.DATE)) + .apply(input, 1); + + assertEquals(1, output.getPositionCount()); + assertEquals(20, output.getInt(0)); + assertEquals(TSDataType.DATE, output.getDataType()); + + Assert.assertThrows( + RuntimeException.class, + () -> + TypeServices.UPDATE_LAST_ROW_SERVICE + .call(Type.fromTsDataType(TSDataType.VECTOR)) + .apply(input, 0)); + } + + @Test + public void testModeValueServiceUsesNativeValues() { + assertModeValueRoundTrip( + TSDataType.BOOLEAN, new BooleanColumn(1, Optional.empty(), new boolean[] {true}), true); + assertModeValueRoundTrip( + TSDataType.INT32, new IntColumn(1, Optional.empty(), new int[] {123}), 123); + assertModeValueRoundTrip( + TSDataType.DATE, new IntColumn(1, Optional.empty(), new int[] {456}, TSDataType.DATE), 456); + assertModeValueRoundTrip( + TSDataType.INT64, new LongColumn(1, Optional.empty(), new long[] {789L}), 789L); + assertModeValueRoundTrip( + TSDataType.TIMESTAMP, new LongColumn(1, Optional.empty(), new long[] {987L}), 987L); + assertModeValueRoundTrip( + TSDataType.FLOAT, new FloatColumn(1, Optional.empty(), new float[] {1.25F}), 1.25F); + assertModeValueRoundTrip( + TSDataType.DOUBLE, new DoubleColumn(1, Optional.empty(), new double[] {2.5}), 2.5); + + Binary binary = new Binary(new byte[] {1, 2, 3}); + Column binaryColumn = new BinaryColumn(1, Optional.empty(), new Binary[] {binary}); + assertModeValueRoundTrip(TSDataType.TEXT, binaryColumn, binary); + assertModeValueRoundTrip(TSDataType.STRING, binaryColumn, binary); + assertModeValueRoundTrip(TSDataType.BLOB, binaryColumn, binary); + } + + // Covers every supported RANGE-frame type and guards native integer overflow and long precision. + @Test + public void testRangeFrameComparatorPreservesNativeArithmetic() { + assertRangeFrameComparator( + TSDataType.INT32, + new IntColumn(3, Optional.empty(), new int[] {10, 12, 8}), + new IntColumn(3, Optional.empty(), new int[] {2, 0, 0})); + assertRangeFrameComparator( + TSDataType.DATE, + new IntColumn(3, Optional.empty(), new int[] {10, 12, 8}, TSDataType.DATE), + new IntColumn(3, Optional.empty(), new int[] {2, 0, 0})); + assertRangeFrameComparator( + TSDataType.INT64, + new LongColumn(3, Optional.empty(), new long[] {10, 12, 8}), + new LongColumn(3, Optional.empty(), new long[] {2, 0, 0})); + assertRangeFrameComparator( + TSDataType.TIMESTAMP, + new LongColumn(3, Optional.empty(), new long[] {10, 12, 8}), + new LongColumn(3, Optional.empty(), new long[] {2, 0, 0})); + assertRangeFrameComparator( + TSDataType.FLOAT, + new FloatColumn(3, Optional.empty(), new float[] {10, 12, 8}), + new FloatColumn(3, Optional.empty(), new float[] {2, 0, 0})); + assertRangeFrameComparator( + TSDataType.DOUBLE, + new DoubleColumn(3, Optional.empty(), new double[] {10, 12, 8}), + new DoubleColumn(3, Optional.empty(), new double[] {2, 0, 0})); + + assertTrue( + compareRangeFrameValues( + TSDataType.INT32, + new IntColumn(2, Optional.empty(), new int[] {Integer.MAX_VALUE, Integer.MIN_VALUE}), + new IntColumn(2, Optional.empty(), new int[] {1, 0}), + TypeServices.RangeFrameOffsetOperation.ADD, + TypeServices.RangeFrameComparison.GREATER_THAN_OR_EQUAL)); + assertFalse( + compareRangeFrameValues( + TSDataType.INT64, + new LongColumn( + 2, Optional.empty(), new long[] {9_007_199_254_740_992L, 9_007_199_254_740_992L}), + new LongColumn(2, Optional.empty(), new long[] {1, 0}), + TypeServices.RangeFrameOffsetOperation.ADD, + TypeServices.RangeFrameComparison.GREATER_THAN_OR_EQUAL)); + } + + private static double convert(final TSDataType dataType, final Column column) { + return TypeServices.NUMERIC_COLUMN_TO_DOUBLE_CONVERTER_SERVICE + .call(Type.fromTsDataType(dataType)) + .create( + () -> + new IllegalStateException( + "supported data type should not use the exception factory")) + .convert(column, 0); + } + + private static void assertModeValueRoundTrip( + final TSDataType dataType, final Column input, final Object expectedValue) { + Type type = Type.fromTsDataType(dataType); + TypeServices.ModeValueService valueService = TypeServices.MODE_VALUE_SERVICE.call(type); + + Object value = valueService.getValue(input, 0); + assertEquals(expectedValue, value); + byte[] bytes = new byte[valueService.calcTypeSize(value)]; + assertEquals(bytes.length, valueService.serialize(value, bytes, 0)); + Object deserializedValue = valueService.deserialize(bytes, 0); + assertEquals(value, deserializedValue); + + ColumnBuilder outputBuilder = type.createColumnBuilder(null, 1); + valueService.write(outputBuilder, deserializedValue); + assertTrue(input.arePositionsEqual(0, outputBuilder.build(), 0)); + } + + private static void assertRangeFrameComparator( + final TSDataType dataType, final Column valueColumn, final Column offsetColumn) { + assertTrue( + compareRangeFrameValues( + dataType, + valueColumn, + offsetColumn, + TypeServices.RangeFrameOffsetOperation.ADD, + TypeServices.RangeFrameComparison.GREATER_THAN_OR_EQUAL)); + assertFalse( + compareRangeFrameValues( + dataType, + valueColumn, + offsetColumn, + TypeServices.RangeFrameOffsetOperation.ADD, + TypeServices.RangeFrameComparison.GREATER_THAN)); + assertTrue( + compareRangeFrameValues( + dataType, + valueColumn, + offsetColumn, + TypeServices.RangeFrameOffsetOperation.SUBTRACT, + TypeServices.RangeFrameComparison.LESS_THAN_OR_EQUAL)); + assertFalse( + compareRangeFrameValues( + dataType, + valueColumn, + offsetColumn, + TypeServices.RangeFrameOffsetOperation.SUBTRACT, + TypeServices.RangeFrameComparison.LESS_THAN)); + } + + private static boolean compareRangeFrameValues( + final TSDataType dataType, + final Column valueColumn, + final Column offsetColumn, + final TypeServices.RangeFrameOffsetOperation offsetOperation, + final TypeServices.RangeFrameComparison comparison) { + final long[] times = new long[valueColumn.getPositionCount()]; + final TsBlock block = + new TsBlock( + new TimeColumn(valueColumn.getPositionCount(), times), valueColumn, offsetColumn); + final Partition partition = + new Partition(Collections.singletonList(block), 0, valueColumn.getPositionCount()); + final ColumnList columnList = new ColumnList(Collections.singletonList(valueColumn)); + return TypeServices.RANGE_FRAME_COMPARATOR_SERVICE + .call(Type.fromTsDataType(dataType)) + .compare( + columnList, + partition, + 0, + offsetOperation == TypeServices.RangeFrameOffsetOperation.ADD + ? 1 + : valueColumn.getPositionCount() - 1, + 1, + offsetOperation, + comparison); + } +} diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml index b615061fc745..3af141267899 100644 --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@ -314,6 +314,11 @@ 1.3.0 test + + org.ow2.asm + asm + test + diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 12f4e374ca7a..fb73485233dc 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -603,7 +603,12 @@ private DataNodeMiscMessages() {} public static final String INPUT_DOUBLE_INFINITY = "The input double value is Infinity"; public static final String BOOLEAN_PARSE_ERROR = "The BOOLEAN should be true/TRUE, false/FALSE or 0/1"; - public static final String UNSUPPORTED_DATA_TYPE_FMT = "Unsupported data type:%s"; + public static final String EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_ARG_REGISTERED_ARG_0EF32FD3 = + "data type is not consistent, input %s, registered %s"; + public static final String EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_ARG_REGISTERED_ARG_BECAUSE_ARG_C8637858 = + "data type is not consistent, input %s, registered %s because %s"; + public static final String UNSUPPORTED_DATA_TYPE_FMT = + "Unsupported data type:%s"; public static final String UNSUPPORTED_DATA_TYPE = "Unsupported data type: "; public static final String AGGREGATE_FUNCTION_NAME_NULL = "AggregateFunction Name must not be null"; diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index b818a52e8387..a0b962545125 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -1544,6 +1544,12 @@ public final class DataNodePipeMessages { public static final String FAILED_TO_UNBIND_FROM_PIPE_TSFILE_TO = "Failed to unbind from pipe tsfile to tablets metrics, pipe map is not empty, pipe: {}"; + // --------------------------------------------------------------------------- + // pipe – PipeRow + // --------------------------------------------------------------------------- + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_FOR_COLUMN_ARG_4C4CCA6D = + "unsupported data type %s for column %s"; + // --------------------------------------------------------------------------- // pipe – AbstractSameTypeNumericOperator // --------------------------------------------------------------------------- diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index bf7a36010030..18618d9d5cd4 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -531,6 +531,18 @@ public final class DataNodeQueryMessages { "TIMESTAMP does not support REGEXP/NOT REGEXP"; public static final String GROUPBYTIME_FILTER_CANNOT_EXIST_IN_VALUE_FILTER = "GroupByTime filter cannot exist in value filter."; + public static final String EXCEPTION_ARG_CANNOT_BE_CAST_TO_ARG_28F0C5FC = + "\"%s\" cannot be cast to [%s]"; + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348 = + "Unsupported data type %s"; + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MAXBY_MINBY_ARG_DD30FAB1 = + "Unsupported data type in MaxBy/MinBy: %s"; + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EQUAL_EVENT_AGGREGATION_ARG_77E085EB = + "Unsupported data type in equal event aggregation : %s"; + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_VARIATION_EVENT_AGGREGATION_ARG_3D0F0388 = + "Unsupported data type in variation event aggregation : %s"; + public static final String EXCEPTION_TIMESTAMP_IN_LIST_LITERAL_CAN_ONLY_BE_LONGLITERAL_DOUBLELITERAL_OR_GENERICLITERAL_ACTUAL_TYPE_ARG_D751BE79 = + "TIMESTAMP IN-list literal can only be LongLiteral, DoubleLiteral, or GenericLiteral; actual type: %s"; public static final String IS_NULL_CAN_BE_PUSHED_DOWN = "IS NULL can be pushed down"; public static final String GROUP_BY_TIME_CANNOT_BE_REVERSED = @@ -1023,6 +1035,8 @@ public final class DataNodeQueryMessages { "Unexpected expression: "; public static final String FAILED_TO_FETCH_SUBQUERY_RESULT = "Failed to Fetch Subquery Result."; + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_FOR_SCALAR_SUBQUERY_RESULT_ARG_D58CBB00 = + "Unsupported data type for scalar subquery result: %s"; // --- Plan / Relational / Planner / Iterative --- @@ -1396,6 +1410,8 @@ public final class DataNodeQueryMessages { "Error occurred during getting UDF access strategy"; public static final String TRANSFORMUTILS_SHOULD_NOT_BE_INSTANTIATED = "TransformUtils should not be instantiated."; + public static final String EXCEPTION_THE_DATA_TYPE_OF_THE_STATE_WINDOW_STRATEGY_IS_NOT_VALID_61F4E273 = + "The data type of the state window strategy is not valid."; // --- Execution / Exchange (additional) --- diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index d809fea19925..d047c915c082 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -603,7 +603,12 @@ private DataNodeMiscMessages() {} public static final String INPUT_DOUBLE_INFINITY = "输入的双精度数值为 Infinity"; public static final String BOOLEAN_PARSE_ERROR = "BOOLEAN 值应为 true/TRUE、false/FALSE 或 0/1"; - public static final String UNSUPPORTED_DATA_TYPE_FMT = "不支持的数据类型：%s"; + public static final String EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_ARG_REGISTERED_ARG_0EF32FD3 = + "数据类型不一致，输入值 %s，注册类型 %s"; + public static final String EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_ARG_REGISTERED_ARG_BECAUSE_ARG_C8637858 = + "数据类型不一致，输入值 %s，注册类型 %s，原因：%s"; + public static final String UNSUPPORTED_DATA_TYPE_FMT = + "不支持的数据类型：%s"; public static final String UNSUPPORTED_DATA_TYPE = "不支持的数据类型："; public static final String AGGREGATE_FUNCTION_NAME_NULL = "聚合函数名称不能为空"; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index fe768a1f0181..ddeae5c35cee 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -1443,6 +1443,12 @@ public final class DataNodePipeMessages { public static final String FAILED_TO_UNBIND_FROM_PIPE_TSFILE_TO = "解绑 from pipe tsfile to tablets metrics, pipe map is not empty, pipe: {} 失败"; + // --------------------------------------------------------------------------- + // pipe – PipeRow + // --------------------------------------------------------------------------- + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_FOR_COLUMN_ARG_4C4CCA6D = + "不支持的数据类型 %s，列名为 %s"; + // --------------------------------------------------------------------------- // pipe – AbstractSameTypeNumericOperator // --------------------------------------------------------------------------- diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index f26f2306be2f..138a4706b833 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -512,6 +512,18 @@ public final class DataNodeQueryMessages { "TIMESTAMP 不支持 REGEXP/NOT REGEXP"; public static final String GROUPBYTIME_FILTER_CANNOT_EXIST_IN_VALUE_FILTER = "GroupByTime 过滤器不能存在于值过滤器中。"; + public static final String EXCEPTION_ARG_CANNOT_BE_CAST_TO_ARG_28F0C5FC = + "\"%s\" 无法转换为 [%s]"; + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348 = + "不支持的数据类型 %s"; + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MAXBY_MINBY_ARG_DD30FAB1 = + "MaxBy/MinBy 不支持的数据类型：%s"; + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EQUAL_EVENT_AGGREGATION_ARG_77E085EB = + "等值事件聚合不支持的数据类型：%s"; + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_VARIATION_EVENT_AGGREGATION_ARG_3D0F0388 = + "变化事件聚合不支持的数据类型：%s"; + public static final String EXCEPTION_TIMESTAMP_IN_LIST_LITERAL_CAN_ONLY_BE_LONGLITERAL_DOUBLELITERAL_OR_GENERICLITERAL_ACTUAL_TYPE_ARG_D751BE79 = + "TIMESTAMP 的 IN 列表字面量只能是 LongLiteral、DoubleLiteral 或 GenericLiteral，实际类型为 %s"; public static final String IS_NULL_CAN_BE_PUSHED_DOWN = "IS NULL 可以下推"; public static final String GROUP_BY_TIME_CANNOT_BE_REVERSED = @@ -1004,6 +1016,8 @@ public final class DataNodeQueryMessages { "意外的表达式："; public static final String FAILED_TO_FETCH_SUBQUERY_RESULT = "获取子查询结果失败。"; + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_FOR_SCALAR_SUBQUERY_RESULT_ARG_D58CBB00 = + "标量子查询结果的数据类型不受支持：%s"; // --- Plan / Relational / Planner / Iterative --- @@ -1377,6 +1391,8 @@ public final class DataNodeQueryMessages { "获取 UDF 访问策略时发生错误"; public static final String TRANSFORMUTILS_SHOULD_NOT_BE_INSTANTIATED = "TransformUtils 不应被实例化。"; + public static final String EXCEPTION_THE_DATA_TYPE_OF_THE_STATE_WINDOW_STRATEGY_IS_NOT_VALID_61F4E273 = + "状态窗口策略的数据类型无效。"; // --- Execution / Exchange (additional) --- diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java index d8692a3573b3..1244879ed425 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -63,7 +63,7 @@ private StorageEngineMessages() {} public static final String FILE_NO_SUCH_TIME_SERIES = "文件中不包含该时间序列 {}。"; // ======================== Resource Control - Disk ======================== - + public static final String ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY = "所有目录已满，切换系统为只读模式。"; public static final String FAILED_TO_PROCESS_FOLDER = "处理目录失败 '"; public static final String FAIL_TO_GET_CANONICAL_PATH = "获取数据目录 {} 的规范路径失败"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 80f9f1f1b30f..a1f02572ba07 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.conf; +import org.apache.iotdb.calc.utils.TypeServices.DefaultEncodingProvider; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.client.property.ClientPoolProperty.DefaultProperty; @@ -82,7 +83,7 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.PIPE_FOLDER_NAME; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; -public class IoTDBConfig { +public class IoTDBConfig implements DefaultEncodingProvider { /* Names of Watermark methods */ public static final String WATERMARK_GROUPED_LSB = "GroupBasedLSBMethod"; @@ -2613,6 +2614,7 @@ void setDefaultDatabaseLevel(int defaultDatabaseLevel, boolean startUp) { this.defaultDatabaseLevel = defaultDatabaseLevel; } + @Override public TSEncoding getDefaultBooleanEncoding() { return defaultBooleanEncoding; } @@ -2625,6 +2627,7 @@ void setDefaultBooleanEncoding(String defaultBooleanEncoding) { this.defaultBooleanEncoding = TSEncoding.valueOf(defaultBooleanEncoding); } + @Override public TSEncoding getDefaultInt32Encoding() { return defaultInt32Encoding; } @@ -2637,6 +2640,7 @@ void setDefaultInt32Encoding(String defaultInt32Encoding) { this.defaultInt32Encoding = TSEncoding.valueOf(defaultInt32Encoding); } + @Override public TSEncoding getDefaultInt64Encoding() { return defaultInt64Encoding; } @@ -2649,6 +2653,7 @@ void setDefaultInt64Encoding(String defaultInt64Encoding) { this.defaultInt64Encoding = TSEncoding.valueOf(defaultInt64Encoding); } + @Override public TSEncoding getDefaultFloatEncoding() { return defaultFloatEncoding; } @@ -2661,6 +2666,7 @@ void setDefaultFloatEncoding(String defaultFloatEncoding) { this.defaultFloatEncoding = TSEncoding.valueOf(defaultFloatEncoding); } + @Override public TSEncoding getDefaultDoubleEncoding() { return defaultDoubleEncoding; } @@ -2673,6 +2679,7 @@ void setDefaultDoubleEncoding(String defaultDoubleEncoding) { this.defaultDoubleEncoding = TSEncoding.valueOf(defaultDoubleEncoding); } + @Override public TSEncoding getDefaultTextEncoding() { return defaultTextEncoding; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 565f0b7832a6..a467e315b9be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.conf; import org.apache.iotdb.calc.exception.QueryProcessException; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.ConfigurationFileUtils; @@ -77,6 +78,7 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.fileSystem.FSType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.FilePathUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -3121,26 +3123,9 @@ public void loadShuffleProps(TrimProperties properties) { /** Get default encode algorithm by data type */ public TSEncoding getDefaultEncodingByType(TSDataType dataType) { - switch (dataType) { - case BOOLEAN: - return conf.getDefaultBooleanEncoding(); - case INT32: - case DATE: - return conf.getDefaultInt32Encoding(); - case INT64: - case TIMESTAMP: - return conf.getDefaultInt64Encoding(); - case FLOAT: - return conf.getDefaultFloatEncoding(); - case DOUBLE: - return conf.getDefaultDoubleEncoding(); - case STRING: - case BLOB: - case OBJECT: - case TEXT: - default: - return conf.getDefaultTextEncoding(); - } + return TypeServices.DEFAULT_ENCODING_BY_TYPE_SERVICE + .call(Type.fromTsDataType(dataType)) + .apply(conf); } // These configurations are received from config node when registering diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeDataTypeTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeDataTypeTransformer.java index ece35eed0f61..7517eb82206a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeDataTypeTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeDataTypeTransformer.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.pipe.event.common.row; -import org.apache.iotdb.db.i18n.DataNodePipeMessages; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.pipe.api.type.Type; import org.apache.tsfile.enums.TSDataType; @@ -39,34 +39,10 @@ public static List transformToPipeDataTypeList(final List tsDa } public static Type transformToPipeDataType(final TSDataType tsDataType) { - return tsDataType == null ? null : getPipeDataType(tsDataType.getType()); - } - - private static Type getPipeDataType(final byte type) { - switch (type) { - case 0: - return Type.BOOLEAN; - case 1: - return Type.INT32; - case 2: - return Type.INT64; - case 3: - return Type.FLOAT; - case 4: - return Type.DOUBLE; - case 5: - return Type.TEXT; - case 8: - return Type.TIMESTAMP; - case 9: - return Type.DATE; - case 10: - return Type.BLOB; - case 11: - return Type.STRING; - default: - throw new IllegalArgumentException(DataNodePipeMessages.INVALID_INPUT + type); - } + return tsDataType == null + ? null + : TypeServices.Pipe.PIPE_DATA_TYPE_TRANSFORMER_SERVICE.call( + org.apache.tsfile.read.common.type.Type.fromTsDataType(tsDataType)); } private PipeDataTypeTransformer() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java index 808fb69bcaa0..c548bee089c5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.event.common.row; import org.apache.iotdb.db.i18n.DataNodePipeMessages; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.pipe.api.access.Row; import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.iotdb.pipe.api.type.Type; @@ -120,31 +121,9 @@ public String getString(final int columnIndex) { @Override public Object getObject(final int columnIndex) { - switch (getDataType(columnIndex)) { - case INT32: - return getInt(columnIndex); - case DATE: - return getDate(columnIndex); - case INT64: - case TIMESTAMP: - return getLong(columnIndex); - case FLOAT: - return getFloat(columnIndex); - case DOUBLE: - return getDouble(columnIndex); - case BOOLEAN: - return getBoolean(columnIndex); - case TEXT: - case BLOB: - case STRING: - return getBinary(columnIndex); - default: - throw new UnsupportedOperationException( - String.format( - DataNodePipeMessages.PIPE_EXCEPTION_UNSUPPORTED_DATA_TYPE_S_FOR_COLUMN_S_9F870C01, - getDataType(columnIndex), - columnNameStringList[columnIndex])); - } + return TypeServices.Pipe.PIPE_ROW_OBJECT_GETTER_SERVICE + .call(org.apache.tsfile.read.common.type.Type.fromTsDataType(valueColumnTypes[columnIndex])) + .get(this, columnIndex); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletUtils.java index 1e6a5e2cadf4..8d341fa9b9d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletUtils.java @@ -24,14 +24,13 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; -import java.time.LocalDate; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -167,38 +166,12 @@ public static void putValue( final int columnIndex, final TSDataType dataType, final Object value) { - switch (dataType) { - case BOOLEAN: - ((boolean[]) tablet.getValues()[columnIndex])[rowIndex] = (Boolean) value; - break; - case INT32: - ((int[]) tablet.getValues()[columnIndex])[rowIndex] = (Integer) value; - break; - case DATE: - ((LocalDate[]) tablet.getValues()[columnIndex])[rowIndex] = (LocalDate) value; - break; - case INT64: - case TIMESTAMP: - ((long[]) tablet.getValues()[columnIndex])[rowIndex] = (Long) value; - break; - case FLOAT: - ((float[]) tablet.getValues()[columnIndex])[rowIndex] = (Float) value; - break; - case DOUBLE: - ((double[]) tablet.getValues()[columnIndex])[rowIndex] = (Double) value; - break; - case TEXT: - case BLOB: - case STRING: - ((Binary[]) tablet.getValues()[columnIndex])[rowIndex] = toBinary(value); - break; - default: - throw new UnSupportedDataTypeException(DataNodePipeMessages.UNSUPPORTED + dataType); - } + Type type = Type.fromTsDataType(dataType); + type.addValue(rowIndex, value, tablet.getValues()[columnIndex]); unmarkNullValue(tablet, rowIndex, columnIndex); } - private static void unmarkNullValue( + public static void unmarkNullValue( final Tablet tablet, final int rowIndex, final int columnIndex) { final BitMap[] bitMaps = tablet.getBitMaps(); if (Objects.nonNull(bitMaps) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java index 545a587aec49..7a60e99678a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.pipe.api.access.Row; import org.apache.iotdb.pipe.api.collector.RowCollector; import org.apache.iotdb.pipe.api.collector.TabletCollector; @@ -36,17 +37,14 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.StringArrayDeviceID; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; -import org.apache.tsfile.utils.DateUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -62,8 +60,6 @@ public abstract class TabletInsertionEventParser { private static final Logger LOGGER = LoggerFactory.getLogger(TabletInsertionEventParser.class); - private static final LocalDate EMPTY_LOCALDATE = LocalDate.of(1000, 1, 1); - protected final PipeTaskMeta pipeTaskMeta; // used to report progress protected final EnrichedEvent sourceEvent; // used to report progress and filter value columns by time range @@ -477,157 +473,14 @@ private static Object filterValueColumnsByRowIndexList( final boolean isSingleOriginValueColumn, final BitMap originNullValueColumnBitmap, final BitMap nullValueColumnBitmap /* output parameters */) { - switch (type) { - case INT32: - { - final int[] intValueColumns = - isSingleOriginValueColumn - ? new int[] {(int) originValueColumn} - : (int[]) originValueColumn; - final int[] valueColumns = new int[rowIndexList.size()]; - for (int i = 0; i < rowIndexList.size(); ++i) { - if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { - valueColumns[i] = 0; - nullValueColumnBitmap.mark(i); - } else { - valueColumns[i] = intValueColumns[rowIndexList.get(i)]; - } - } - return valueColumns; - } - case DATE: - { - // Always store 'LocalDate[]' to help convert to tablet - final LocalDate[] valueColumns = new LocalDate[rowIndexList.size()]; - if (isSingleOriginValueColumn && originValueColumn instanceof LocalDate - || !isSingleOriginValueColumn && originValueColumn instanceof LocalDate[]) { - // For tablet - final LocalDate[] dateValueColumns = - isSingleOriginValueColumn - ? new LocalDate[] {(LocalDate) originValueColumn} - : (LocalDate[]) originValueColumn; - - for (int i = 0; i < rowIndexList.size(); ++i) { - if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { - valueColumns[i] = EMPTY_LOCALDATE; - nullValueColumnBitmap.mark(i); - } else { - valueColumns[i] = dateValueColumns[rowIndexList.get(i)]; - } - } - } else { - // For insertRowNode / insertTabletNode - final int[] intValueColumns = - isSingleOriginValueColumn - ? new int[] {(int) originValueColumn} - : (int[]) originValueColumn; - for (int i = 0; i < rowIndexList.size(); ++i) { - if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { - valueColumns[i] = EMPTY_LOCALDATE; - nullValueColumnBitmap.mark(i); - } else { - valueColumns[i] = - DateUtils.parseIntToLocalDate(intValueColumns[rowIndexList.get(i)]); - } - } - } - return valueColumns; - } - case INT64: - case TIMESTAMP: - { - final long[] longValueColumns = - isSingleOriginValueColumn - ? new long[] {(long) originValueColumn} - : (long[]) originValueColumn; - final long[] valueColumns = new long[rowIndexList.size()]; - for (int i = 0; i < rowIndexList.size(); ++i) { - if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { - valueColumns[i] = 0L; - nullValueColumnBitmap.mark(i); - } else { - valueColumns[i] = longValueColumns[rowIndexList.get(i)]; - } - } - return valueColumns; - } - case FLOAT: - { - final float[] floatValueColumns = - isSingleOriginValueColumn - ? new float[] {(float) originValueColumn} - : (float[]) originValueColumn; - final float[] valueColumns = new float[rowIndexList.size()]; - for (int i = 0; i < rowIndexList.size(); ++i) { - if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { - valueColumns[i] = 0F; - nullValueColumnBitmap.mark(i); - } else { - valueColumns[i] = floatValueColumns[rowIndexList.get(i)]; - } - } - return valueColumns; - } - case DOUBLE: - { - final double[] doubleValueColumns = - isSingleOriginValueColumn - ? new double[] {(double) originValueColumn} - : (double[]) originValueColumn; - final double[] valueColumns = new double[rowIndexList.size()]; - for (int i = 0; i < rowIndexList.size(); ++i) { - if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { - valueColumns[i] = 0D; - nullValueColumnBitmap.mark(i); - } else { - valueColumns[i] = doubleValueColumns[rowIndexList.get(i)]; - } - } - return valueColumns; - } - case BOOLEAN: - { - final boolean[] booleanValueColumns = - isSingleOriginValueColumn - ? new boolean[] {(boolean) originValueColumn} - : (boolean[]) originValueColumn; - final boolean[] valueColumns = new boolean[rowIndexList.size()]; - for (int i = 0; i < rowIndexList.size(); ++i) { - if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { - valueColumns[i] = false; - nullValueColumnBitmap.mark(i); - } else { - valueColumns[i] = booleanValueColumns[rowIndexList.get(i)]; - } - } - return valueColumns; - } - case TEXT: - case BLOB: - case STRING: - { - final Binary[] binaryValueColumns = - isSingleOriginValueColumn - ? new Binary[] {(Binary) originValueColumn} - : (Binary[]) originValueColumn; - final Binary[] valueColumns = new Binary[rowIndexList.size()]; - for (int i = 0; i < rowIndexList.size(); ++i) { - if (Objects.isNull(binaryValueColumns[rowIndexList.get(i)]) - || Objects.isNull(binaryValueColumns[rowIndexList.get(i)].getValues()) - || isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { - valueColumns[i] = Binary.EMPTY_VALUE; - nullValueColumnBitmap.mark(i); - } else { - valueColumns[i] = new Binary(binaryValueColumns[rowIndexList.get(i)].getValues()); - } - } - return valueColumns; - } - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodePipeMessages.PIPE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, type)); - } + return TypeServices.Pipe.PIPE_TABLET_VALUE_COLUMN_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .filter( + originValueColumn, + rowIndexList, + isSingleOriginValueColumn, + originNullValueColumnBitmap, + nullValueColumnBitmap); } private void fillNullValue( @@ -640,50 +493,16 @@ private void fillNullValue( if (Objects.isNull(type)) { return; } - switch (type) { - case TIMESTAMP: - case INT64: - valueColumns[columnIndex] = new long[rowSize]; - break; - case INT32: - valueColumns[columnIndex] = new int[rowSize]; - break; - case DOUBLE: - valueColumns[columnIndex] = new double[rowSize]; - break; - case FLOAT: - valueColumns[columnIndex] = new float[rowSize]; - break; - case BOOLEAN: - valueColumns[columnIndex] = new boolean[rowSize]; - break; - case DATE: - final LocalDate[] dates = new LocalDate[rowSize]; - Arrays.fill(dates, EMPTY_LOCALDATE); - valueColumns[columnIndex] = dates; - break; - case TEXT: - case BLOB: - case STRING: - final Binary[] columns = new Binary[rowSize]; - Arrays.fill(columns, Binary.EMPTY_VALUE); - valueColumns[columnIndex] = columns; - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodePipeMessages.PIPE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, type)); - } + valueColumns[columnIndex] = + TypeServices.StorageEngine.EMPTY_TABLET_COLUMN_FACTORY_SERVICE + .call(Type.fromTsDataType(type)) + .apply(rowSize); } private static BitMap getBitMap(final BitMap[] bitMaps, final int index) { return Objects.nonNull(bitMaps) && index < bitMaps.length ? bitMaps[index] : null; } - private static boolean isNullValue(final BitMap bitMap, final int rowIndex) { - return Objects.nonNull(bitMap) && bitMap.isMarked(rowIndex); - } - //////////////////////////// process //////////////////////////// public abstract List processRowByRow( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java index f51d4d73b43a..4fdb6f4e4a50 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java @@ -33,6 +33,7 @@ import org.apache.tsfile.file.metadata.enums.EncryptionType; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.chunk.AbstractChunkReader; import org.apache.tsfile.read.reader.page.LazyLoadPageData; import org.apache.tsfile.read.reader.page.PageReader; @@ -252,26 +253,7 @@ private static long estimateSingleValueArrayMemoryUsageInBytes( return 0; } - switch (dataType) { - case BOOLEAN: - return RamUsageEstimator.sizeOfBooleanArray(16) * segmentCount; - case INT32: - case DATE: - return RamUsageEstimator.sizeOfIntArray(16) * segmentCount; - case INT64: - case TIMESTAMP: - return RamUsageEstimator.sizeOfLongArray(16) * segmentCount; - case FLOAT: - return RamUsageEstimator.sizeOfFloatArray(16) * segmentCount; - case DOUBLE: - return RamUsageEstimator.sizeOfDoubleArray(16) * segmentCount; - case TEXT: - case BLOB: - case STRING: - return RamUsageEstimator.sizeOfObjectArray(16) * segmentCount; - default: - return 0; - } + return Type.fromTsDataType(dataType).estimateArraySize(16) * segmentCount; } private static long estimateVectorValueMemoryUsageInBytes( @@ -291,24 +273,7 @@ private static long estimateVectorValueMemoryUsageInBytes( } private static long estimateTsPrimitiveTypeValueMemoryUsageInBytes(final TSDataType dataType) { - switch (dataType) { - case BOOLEAN: - return 1; - case INT32: - case DATE: - case FLOAT: - return Integer.BYTES; - case INT64: - case TIMESTAMP: - case DOUBLE: - return Long.BYTES; - case TEXT: - case BLOB: - case STRING: - return RamUsageEstimator.NUM_BYTES_OBJECT_REF; - default: - return 0; - } + return Type.fromTsDataType(dataType).estimateValueSize(); } ///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java index e18494ea492c..aab44bc6882a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java @@ -41,6 +41,7 @@ import org.apache.iotdb.db.pipe.event.common.tsfile.parser.TsFileInsertionEventParserMemoryManager; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.util.ModsOperationUtil; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; @@ -58,16 +59,15 @@ import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.BufferedTsFileInput; import org.apache.tsfile.read.reader.IChunkReader; import org.apache.tsfile.read.reader.chunk.AlignedChunkReader; import org.apache.tsfile.read.reader.chunk.ChunkReader; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -581,79 +581,21 @@ private boolean putValueToColumns(final BatchData data, final Tablet tablet, fin if (data.getDataType() == TSDataType.VECTOR) { for (int i = 0; i < tablet.getSchemas().size(); ++i) { final TsPrimitiveType primitiveType = data.getVector()[i]; + final TSDataType dataType = tablet.getSchemas().get(i).getType(); if (Objects.isNull(primitiveType) || ModsOperationUtil.isDelete(data.currentTime(), modsInfos.get(i))) { - switch (tablet.getSchemas().get(i).getType()) { - case TEXT: - case BLOB: - case STRING: - PipeTabletUtils.putValue( - tablet, rowIndex, i, tablet.getSchemas().get(i).getType(), Binary.EMPTY_VALUE); + if (dataType.isBinary()) { + ((Binary[]) tablet.getValues()[i])[rowIndex] = Binary.EMPTY_VALUE; } PipeTabletUtils.markNullValue(tablet, rowIndex, i); continue; } isNeedFillTime = true; - switch (tablet.getSchemas().get(i).getType()) { - case BOOLEAN: - PipeTabletUtils.putValue( - tablet, - rowIndex, - i, - tablet.getSchemas().get(i).getType(), - primitiveType.getBoolean()); - break; - case INT32: - PipeTabletUtils.putValue( - tablet, rowIndex, i, tablet.getSchemas().get(i).getType(), primitiveType.getInt()); - break; - case DATE: - PipeTabletUtils.putValue( - tablet, - rowIndex, - i, - tablet.getSchemas().get(i).getType(), - DateUtils.parseIntToLocalDate(primitiveType.getInt())); - break; - case INT64: - case TIMESTAMP: - PipeTabletUtils.putValue( - tablet, rowIndex, i, tablet.getSchemas().get(i).getType(), primitiveType.getLong()); - break; - case FLOAT: - PipeTabletUtils.putValue( - tablet, - rowIndex, - i, - tablet.getSchemas().get(i).getType(), - primitiveType.getFloat()); - break; - case DOUBLE: - PipeTabletUtils.putValue( - tablet, - rowIndex, - i, - tablet.getSchemas().get(i).getType(), - primitiveType.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - final Binary binary = primitiveType.getBinary(); - PipeTabletUtils.putValue( - tablet, - rowIndex, - i, - tablet.getSchemas().get(i).getType(), - Objects.isNull(binary) || Objects.isNull(binary.getValues()) - ? Binary.EMPTY_VALUE - : binary); - break; - default: - throw new UnSupportedDataTypeException( - DataNodePipeMessages.UNSUPPORTED + primitiveType.getDataType()); - } + TypeServices.Pipe.PIPE_TS_PRIMITIVE_TABLET_VALUE_WRITER_SERVICE + .call(Type.fromTsDataType(dataType)) + .write(primitiveType, tablet.getValues()[i], rowIndex); + PipeTabletUtils.unmarkNullValue(tablet, rowIndex, i); } } else { if (!modsInfos.isEmpty() @@ -662,53 +604,11 @@ private boolean putValueToColumns(final BatchData data, final Tablet tablet, fin } isNeedFillTime = true; - switch (tablet.getSchemas().get(0).getType()) { - case BOOLEAN: - PipeTabletUtils.putValue( - tablet, rowIndex, 0, tablet.getSchemas().get(0).getType(), data.getBoolean()); - break; - case INT32: - PipeTabletUtils.putValue( - tablet, rowIndex, 0, tablet.getSchemas().get(0).getType(), data.getInt()); - break; - case DATE: - PipeTabletUtils.putValue( - tablet, - rowIndex, - 0, - tablet.getSchemas().get(0).getType(), - DateUtils.parseIntToLocalDate(data.getInt())); - break; - case INT64: - case TIMESTAMP: - PipeTabletUtils.putValue( - tablet, rowIndex, 0, tablet.getSchemas().get(0).getType(), data.getLong()); - break; - case FLOAT: - PipeTabletUtils.putValue( - tablet, rowIndex, 0, tablet.getSchemas().get(0).getType(), data.getFloat()); - break; - case DOUBLE: - PipeTabletUtils.putValue( - tablet, rowIndex, 0, tablet.getSchemas().get(0).getType(), data.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - final Binary binary = data.getBinary(); - PipeTabletUtils.putValue( - tablet, - rowIndex, - 0, - tablet.getSchemas().get(0).getType(), - Objects.isNull(binary) || Objects.isNull(binary.getValues()) - ? Binary.EMPTY_VALUE - : binary); - break; - default: - throw new UnSupportedDataTypeException( - DataNodePipeMessages.UNSUPPORTED + data.getDataType()); - } + final TSDataType dataType = tablet.getSchemas().get(0).getType(); + TypeServices.Pipe.PIPE_BATCH_DATA_TABLET_VALUE_WRITER_SERVICE + .call(Type.fromTsDataType(dataType)) + .write(data, tablet.getValues()[0], rowIndex); + PipeTabletUtils.unmarkNullValue(tablet, rowIndex, 0); } return isNeedFillTime; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java index 153a6d5288da..142fbaa88d89 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.table; +import org.apache.iotdb.calc.utils.TypeServices; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeOutOfMemoryCriticalException; import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.db.exception.load.LoadRuntimeOutOfMemoryException; @@ -44,15 +45,14 @@ import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.controller.IMetadataQuerier; import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.tsfile.read.reader.IChunkReader; import org.apache.tsfile.read.reader.chunk.TableChunkReader; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -510,6 +510,7 @@ private boolean fillMeasurementValueColumns( boolean hasNonDeletedField = dataTypeList.size() == deviceIdSize; for (int i = deviceIdSize, size = dataTypeList.size(); i < size; i++) { + final TSDataType dataType = dataTypeList.get(i); final TsPrimitiveType primitiveType = i - deviceIdSize < primitiveTypes.length ? primitiveTypes[i - deviceIdSize] : null; final boolean isDeleted = ModsOperationUtil.isDelete(data.currentTime(), modsInfoList.get(i)); @@ -517,64 +518,21 @@ private boolean fillMeasurementValueColumns( hasNonDeletedField = true; } if (primitiveType == null || isDeleted) { - switch (dataTypeList.get(i)) { - case TEXT: - case BLOB: - case STRING: - PipeTabletUtils.putValue(tablet, rowIndex, i, dataTypeList.get(i), Binary.EMPTY_VALUE); + if (dataType.isTextStringOrBlob()) { + PipeTabletUtils.putValue(tablet, rowIndex, i, dataType, Binary.EMPTY_VALUE); } PipeTabletUtils.markNullValue(tablet, rowIndex, i); continue; } needFillTime = true; - - switch (dataTypeList.get(i)) { - case BOOLEAN: - PipeTabletUtils.putValue( - tablet, rowIndex, i, dataTypeList.get(i), primitiveType.getBoolean()); - break; - case INT32: - PipeTabletUtils.putValue( - tablet, rowIndex, i, dataTypeList.get(i), primitiveType.getInt()); - break; - case DATE: - PipeTabletUtils.putValue( - tablet, - rowIndex, - i, - dataTypeList.get(i), - DateUtils.parseIntToLocalDate(primitiveType.getInt())); - break; - case INT64: - case TIMESTAMP: - PipeTabletUtils.putValue( - tablet, rowIndex, i, dataTypeList.get(i), primitiveType.getLong()); - break; - case FLOAT: - PipeTabletUtils.putValue( - tablet, rowIndex, i, dataTypeList.get(i), primitiveType.getFloat()); - break; - case DOUBLE: - PipeTabletUtils.putValue( - tablet, rowIndex, i, dataTypeList.get(i), primitiveType.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - Binary binary = primitiveType.getBinary(); - PipeTabletUtils.putValue( - tablet, - rowIndex, - i, - dataTypeList.get(i), - Objects.isNull(binary) || Objects.isNull(binary.getValues()) - ? Binary.EMPTY_VALUE - : binary); - break; - default: - throw new UnSupportedDataTypeException( - DataNodePipeMessages.UNSUPPORTED + primitiveType.getDataType()); - } + PipeTabletUtils.putValue( + tablet, + rowIndex, + i, + dataType, + TypeServices.PRIMITIVE_TYPE_VALUE_EXTRACTOR_SERVICE + .call(Type.fromTsDataType(dataType)) + .apply(primitiveType)); } return needFillTime || hasNonDeletedField; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java index d9140659e264..f549ad182d95 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowOutput; import org.apache.iotdb.db.pipe.processor.aggregate.window.processor.AbstractWindowingProcessor; import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.pipe.api.PipeProcessor; import org.apache.iotdb.pipe.api.access.Row; import org.apache.iotdb.pipe.api.annotation.TreeModel; @@ -60,17 +61,15 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.schema.MeasurementSchema; import java.io.IOException; import java.nio.ByteBuffer; -import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -470,56 +469,7 @@ private Map> processRow( synchronized (stateReference) { final TimeSeriesRuntimeState state = stateReference.get(); try { - switch (row.getDataType(index)) { - case BOOLEAN: - result = - state.updateWindows( - timestamp, row.getBoolean(index), outputMinReportIntervalMilliseconds); - break; - case INT32: - result = - state.updateWindows( - timestamp, row.getInt(index), outputMinReportIntervalMilliseconds); - break; - case DATE: - result = - state.updateWindows( - timestamp, row.getDate(index), outputMinReportIntervalMilliseconds); - break; - case INT64: - case TIMESTAMP: - result = - state.updateWindows( - timestamp, row.getLong(index), outputMinReportIntervalMilliseconds); - break; - case FLOAT: - result = - state.updateWindows( - timestamp, row.getFloat(index), outputMinReportIntervalMilliseconds); - break; - case DOUBLE: - result = - state.updateWindows( - timestamp, row.getDouble(index), outputMinReportIntervalMilliseconds); - break; - case TEXT: - case STRING: - result = - state.updateWindows( - timestamp, row.getString(index), outputMinReportIntervalMilliseconds); - break; - case BLOB: - case OBJECT: - result = - state.updateWindows( - timestamp, row.getBinary(index), outputMinReportIntervalMilliseconds); - break; - default: - throw new UnsupportedOperationException( - String.format( - DataNodePipeMessages.PIPE_EXCEPTION_THE_TYPE_S_IS_NOT_SUPPORTED_E1A6F05D, - row.getDataType(index))); - } + result = state.updateWindows(timestamp, row, index, outputMinReportIntervalMilliseconds); if (Objects.nonNull(result)) { collectWindowOutputs(result.getLeft(), timeSeries, rowCollector); if (Objects.nonNull(result.getRight())) { @@ -662,6 +612,7 @@ public void collectWindowOutputs( final MeasurementSchema[] measurementSchemaList = new MeasurementSchema[columnNameStringList.length]; final TSDataType[] valueColumnTypes = new TSDataType[columnNameStringList.length]; + final Type[] valueTypes = new Type[columnNameStringList.length]; final Object[] valueColumns = new Object[columnNameStringList.length]; final BitMap[] bitMaps = new BitMap[columnNameStringList.length]; @@ -684,90 +635,19 @@ public void collectWindowOutputs( measurementSchemaList[columnIndex] = new MeasurementSchema( columnNameStringList[columnIndex], valueColumnTypes[columnIndex]); - switch (valueColumnTypes[columnIndex]) { - case BOOLEAN: - valueColumns[columnIndex] = new boolean[distinctOutputs.size()]; - break; - case INT32: - valueColumns[columnIndex] = new int[distinctOutputs.size()]; - break; - case DATE: - valueColumns[columnIndex] = new LocalDate[distinctOutputs.size()]; - break; - case INT64: - case TIMESTAMP: - valueColumns[columnIndex] = new long[distinctOutputs.size()]; - break; - case FLOAT: - valueColumns[columnIndex] = new float[distinctOutputs.size()]; - break; - case DOUBLE: - valueColumns[columnIndex] = new double[distinctOutputs.size()]; - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - valueColumns[columnIndex] = new Binary[distinctOutputs.size()]; - break; - default: - throw new UnsupportedOperationException( - String.format( - DataNodePipeMessages - .PIPE_EXCEPTION_THE_OUTPUT_TABLET_DOES_NOT_SUPPORT_COLUMN_TYPE_S_62F3845C, - valueColumnTypes[columnIndex])); - } + valueTypes[columnIndex] = getOutputTabletType(valueColumnTypes[columnIndex]); + valueColumns[columnIndex] = + TypeServices.Pipe.AGGREGATE_TABLET_COLUMN_ALLOCATOR_SERVICE + .call(valueTypes[columnIndex]) + .apply(distinctOutputs.size()); } // Fill in values - switch (valueColumnTypes[columnIndex]) { - case BOOLEAN: - ((boolean[]) valueColumns[columnIndex])[rowIndex] = - (boolean) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); - break; - case INT32: - ((int[]) valueColumns[columnIndex])[rowIndex] = - (int) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); - break; - case DATE: - ((LocalDate[]) valueColumns[columnIndex])[rowIndex] = - (LocalDate) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); - break; - case INT64: - case TIMESTAMP: - ((long[]) valueColumns[columnIndex])[rowIndex] = - (long) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); - break; - case FLOAT: - ((float[]) valueColumns[columnIndex])[rowIndex] = - (float) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); - break; - case DOUBLE: - ((double[]) valueColumns[columnIndex])[rowIndex] = - (double) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); - break; - case TEXT: - case STRING: - ((Binary[]) valueColumns[columnIndex])[rowIndex] = - aggregatedResults.get(columnNameStringList[columnIndex]).getRight() - instanceof Binary - ? (Binary) aggregatedResults.get(columnNameStringList[columnIndex]).getRight() - : new Binary( - (String) - aggregatedResults.get(columnNameStringList[columnIndex]).getRight(), - TSFileConfig.STRING_CHARSET); - break; - case BLOB: - case OBJECT: - ((Binary[]) valueColumns[columnIndex])[rowIndex] = - (Binary) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); - break; - default: - throw new UnsupportedOperationException( - String.format( - DataNodePipeMessages - .PIPE_EXCEPTION_THE_OUTPUT_TABLET_DOES_NOT_SUPPORT_COLUMN_TYPE_S_62F3845C, - valueColumnTypes[columnIndex])); - } + TypeServices.Pipe.AGGREGATE_TABLET_COLUMN_VALUE_WRITER_SERVICE + .call(valueTypes[columnIndex]) + .write( + valueColumns[columnIndex], + rowIndex, + aggregatedResults.get(columnNameStringList[columnIndex]).getRight()); } else { bitMaps[columnIndex].mark(rowIndex); } @@ -862,6 +742,15 @@ public void collectWindowOutputs( } } + private static Type getOutputTabletType(final TSDataType dataType) { + try { + return Type.fromTsDataType(dataType); + } catch (final UnsupportedOperationException ignored) { + throw new UnsupportedOperationException( + String.format(DataNodePipeMessages.UNSUPPORTED_OUTPUT_DATATYPE_FMT, dataType)); + } + } + @Override public void close() throws Exception { if (Objects.nonNull(pipeName) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java index 2cb9a4146903..cc8bfca77611 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java @@ -25,8 +25,12 @@ import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowOutput; import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowState; import org.apache.iotdb.db.pipe.processor.aggregate.window.processor.AbstractWindowingProcessor; +import org.apache.iotdb.db.utils.TypeServices; +import org.apache.iotdb.pipe.api.access.Row; import org.apache.iotdb.pipe.api.type.Binary; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -61,6 +65,7 @@ public class TimeSeriesRuntimeState { // Variables to avoid "new" operation private final List outputList = new ArrayList<>(); + private TypeServices.Pipe.AggregateRowValueUpdater rowValueUpdater; public TimeSeriesRuntimeState( final Map aggregatorOutputName2OperatorMap, @@ -74,6 +79,22 @@ public TimeSeriesRuntimeState( this.windowingProcessor = windowingProcessor; } + public Pair, Pair> updateWindows( + final long timestamp, + final Row row, + final int columnIndex, + final long outputMinReportIntervalMilliseconds) + throws IOException { + if (rowValueUpdater == null) { + rowValueUpdater = + TypeServices.Pipe.AGGREGATE_ROW_VALUE_UPDATER_SERVICE.call( + Type.fromTsDataType( + TSDataType.getTsDataType(row.getDataType(columnIndex).getType()))); + } + return rowValueUpdater.update( + this, timestamp, row, columnIndex, outputMinReportIntervalMilliseconds); + } + // The following "updateWindows" are the same except for the input value types public Pair, Pair> updateWindows( final long timestamp, final boolean value, final long outputMinReportIntervalMilliseconds) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java index 0a39312e69f7..c0bad1319973 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java @@ -19,20 +19,20 @@ package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult; -import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.db.i18n.DataNodePipeMessages; -import org.apache.iotdb.pipe.api.type.Binary; -import org.apache.iotdb.rpc.RpcUtils; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.BytesUtils; -import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Pair; -import java.time.LocalDate; -import java.time.ZoneId; import java.util.Map; +import static org.apache.iotdb.db.utils.TypeServices.Pipe.CUSTOMIZED_INTERMEDIATE_RESULT_TO_DOUBLE_SERVICE; +import static org.apache.iotdb.db.utils.TypeServices.Pipe.CUSTOMIZED_INTERMEDIATE_RESULT_TO_FLOAT_SERVICE; +import static org.apache.iotdb.db.utils.TypeServices.Pipe.CUSTOMIZED_INTERMEDIATE_RESULT_TO_INT_SERVICE; +import static org.apache.iotdb.db.utils.TypeServices.Pipe.CUSTOMIZED_INTERMEDIATE_RESULT_TO_LONG_SERVICE; +import static org.apache.iotdb.db.utils.TypeServices.Pipe.CUSTOMIZED_INTERMEDIATE_RESULT_TO_STRING_SERVICE; + public class CustomizedReadableIntermediateResults { private final Map> intermediateResults; @@ -55,117 +55,33 @@ public boolean getBoolean(final String key) { public int getInt(final String key) { final Pair typeResultPair = intermediateResults.get(key); - final TSDataType type = typeResultPair.getLeft(); - final Object value = typeResultPair.getRight(); - switch (type) { - case INT32: - return (int) value; - case DATE: - return DateUtils.parseDateExpressionToInt((LocalDate) value); - case INT64: - case TIMESTAMP: - return (int) (long) value; - case FLOAT: - return (int) (float) value; - case DOUBLE: - return (int) (double) value; - case TEXT: - case BLOB: - case BOOLEAN: - case STRING: - default: - throw new UnsupportedOperationException( - String.format( - DataNodePipeMessages.PIPE_EXCEPTION_THE_TYPE_S_CANNOT_BE_CASTED_TO_INT_659069CC, - typeResultPair.getLeft())); - } + return CUSTOMIZED_INTERMEDIATE_RESULT_TO_INT_SERVICE + .call(getType(typeResultPair.getLeft(), "int")) + .applyAsInt(typeResultPair.getRight()); } public long getLong(final String key) { final Pair typeResultPair = intermediateResults.get(key); - final TSDataType type = typeResultPair.getLeft(); - final Object value = typeResultPair.getRight(); - switch (type) { - case INT32: - return (int) value; - case DATE: - return DateUtils.parseDateExpressionToInt((LocalDate) value); - case INT64: - case TIMESTAMP: - return (long) value; - case FLOAT: - return (long) (float) value; - case DOUBLE: - return (long) (double) value; - case BOOLEAN: - case STRING: - case TEXT: - case BLOB: - default: - throw new UnsupportedOperationException( - String.format( - DataNodePipeMessages.PIPE_EXCEPTION_THE_TYPE_S_CANNOT_BE_CASTED_TO_LONG_2D206561, - typeResultPair.getLeft())); - } + return CUSTOMIZED_INTERMEDIATE_RESULT_TO_LONG_SERVICE + .call(getType(typeResultPair.getLeft(), "long")) + .applyAsLong(typeResultPair.getRight()); } public float getFloat(final String key) { final Pair typeResultPair = intermediateResults.get(key); - final TSDataType type = typeResultPair.getLeft(); - final Object value = typeResultPair.getRight(); - switch (type) { - case INT32: - return (int) value; - case DATE: - return DateUtils.parseDateExpressionToInt((LocalDate) value); - case INT64: - case TIMESTAMP: - return (long) value; - case FLOAT: - return (float) value; - case DOUBLE: - return (float) (double) value; - case TEXT: - case BLOB: - case BOOLEAN: - case STRING: - default: - throw new UnsupportedOperationException( - String.format( - DataNodePipeMessages.PIPE_EXCEPTION_THE_TYPE_S_CANNOT_BE_CASTED_TO_FLOAT_C15A8A95, - typeResultPair.getLeft())); - } + return CUSTOMIZED_INTERMEDIATE_RESULT_TO_FLOAT_SERVICE + .call(getType(typeResultPair.getLeft(), "float")) + .apply(typeResultPair.getRight()); } public double getDouble(final String key) { final Pair typeResultPair = intermediateResults.get(key); - final TSDataType type = typeResultPair.getLeft(); - final Object value = typeResultPair.getRight(); - switch (type) { - case INT32: - return (int) value; - case DATE: - return DateUtils.parseDateExpressionToInt((LocalDate) value); - case INT64: - case TIMESTAMP: - return (long) value; - case FLOAT: - return (float) value; - case DOUBLE: - return (double) value; - case BOOLEAN: - case STRING: - case TEXT: - case BLOB: - default: - throw new UnsupportedOperationException( - String.format( - DataNodePipeMessages.PIPE_EXCEPTION_THE_TYPE_S_CANNOT_BE_CASTED_TO_DOUBLE_E577C0D7, - typeResultPair.getLeft())); - } + return CUSTOMIZED_INTERMEDIATE_RESULT_TO_DOUBLE_SERVICE + .call(getType(typeResultPair.getLeft(), "double")) + .applyAsDouble(typeResultPair.getRight()); } // Note: This method will cast any decimal types to string without throwing @@ -173,37 +89,17 @@ public double getDouble(final String key) { public String getString(final String key) { final Pair typeResultPair = intermediateResults.get(key); - final TSDataType type = typeResultPair.getLeft(); - final Object value = typeResultPair.getRight(); - switch (type) { - case BOOLEAN: - return Boolean.toString((boolean) value); - case INT32: - return Integer.toString((int) value); - case DATE: - return ((LocalDate) value).toString(); - case INT64: - return Long.toString((long) value); - case TIMESTAMP: - return RpcUtils.formatDatetime( - RpcUtils.DEFAULT_TIME_FORMAT, - CommonDescriptor.getInstance().getConfig().getTimestampPrecision(), - (long) value, - ZoneId.systemDefault()); - case FLOAT: - return Float.toString((float) value); - case DOUBLE: - return Double.toString((double) value); - case TEXT: - case STRING: - return (String) value; - case BLOB: - return BytesUtils.parseBlobByteArrayToString(((Binary) value).getValues()); - default: - throw new UnsupportedOperationException( - String.format( - DataNodePipeMessages.PIPE_EXCEPTION_THE_TYPE_S_CANNOT_BE_CASTED_TO_STRING_34983FBD, - typeResultPair.getLeft())); + return CUSTOMIZED_INTERMEDIATE_RESULT_TO_STRING_SERVICE + .call(getType(typeResultPair.getLeft(), "string")) + .apply(typeResultPair.getRight()); + } + + private static Type getType(final TSDataType dataType, final String targetType) { + try { + return Type.fromTsDataType(dataType); + } catch (final UnsupportedOperationException ignored) { + throw new UnsupportedOperationException( + String.format("The type %s cannot be casted to %s.", dataType, targetType)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java index f3a609ee4905..bf6d4d65bc1b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java @@ -21,11 +21,12 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.IntermediateResultOperator; +import org.apache.iotdb.db.utils.TypeServices.Pipe.SameTypeNumericOperatorStrategy; import org.apache.iotdb.pipe.api.type.Binary; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; import java.io.IOException; @@ -33,6 +34,8 @@ import java.time.LocalDate; import java.util.Map; +import static org.apache.iotdb.db.utils.TypeServices.Pipe.SAME_TYPE_NUMERIC_OPERATOR_STRATEGY_SERVICE; + /** * {@link AbstractSameTypeNumericOperator} is the parent class of all the operators where there * value type is the same as the input data type, the input data type is a numeric type, and the @@ -120,79 +123,71 @@ public void updateValue(final Binary input, final long timestamp) { @Override public Pair getResult() { - switch (outPutDataType) { - case INT32: - return new Pair<>(TSDataType.INT32, intValue); - case INT64: - return new Pair<>(TSDataType.INT64, longValue); - case FLOAT: - return new Pair<>(TSDataType.FLOAT, floatValue); - case DOUBLE: - return new Pair<>(TSDataType.DOUBLE, doubleValue); - case BLOB: - case TEXT: - case BOOLEAN: - case STRING: - case TIMESTAMP: - case DATE: - default: - return null; + try { + return getStrategy().getResult(this); + } catch (final UnsupportedOperationException ignored) { + return null; } } @Override public void serialize(final DataOutputStream outputStream) throws IOException { outPutDataType.serializeTo(outputStream); - switch (outPutDataType) { - case INT32: - ReadWriteIOUtils.write(intValue, outputStream); - break; - case INT64: - ReadWriteIOUtils.write(longValue, outputStream); - break; - case FLOAT: - ReadWriteIOUtils.write(floatValue, outputStream); - break; - case DOUBLE: - ReadWriteIOUtils.write(doubleValue, outputStream); - break; - case TIMESTAMP: - case DATE: - case BOOLEAN: - case STRING: - case TEXT: - case BLOB: - default: - throw new IOException( - String.format(DataNodePipeMessages.UNSUPPORTED_OUTPUT_DATATYPE_FMT, outPutDataType)); + try { + getStrategy().serialize(this, outputStream); + } catch (final UnsupportedOperationException ignored) { + throw unsupportedOutputDataTypeException(); } } @Override public void deserialize(final ByteBuffer byteBuffer) throws IOException { outPutDataType = TSDataType.deserializeFrom(byteBuffer); - switch (outPutDataType) { - case INT32: - intValue = ReadWriteIOUtils.readInt(byteBuffer); - break; - case INT64: - longValue = ReadWriteIOUtils.readLong(byteBuffer); - break; - case FLOAT: - floatValue = ReadWriteIOUtils.readFloat(byteBuffer); - break; - case DOUBLE: - doubleValue = ReadWriteIOUtils.readDouble(byteBuffer); - break; - case TEXT: - case BLOB: - case BOOLEAN: - case STRING: - case DATE: - case TIMESTAMP: - default: - throw new IOException( - String.format(DataNodePipeMessages.UNSUPPORTED_OUTPUT_DATATYPE_FMT, outPutDataType)); + try { + getStrategy().deserialize(this, byteBuffer); + } catch (final UnsupportedOperationException ignored) { + throw unsupportedOutputDataTypeException(); } } + + private SameTypeNumericOperatorStrategy getStrategy() { + return SAME_TYPE_NUMERIC_OPERATOR_STRATEGY_SERVICE.call(Type.fromTsDataType(outPutDataType)); + } + + private IOException unsupportedOutputDataTypeException() { + return new IOException( + String.format(DataNodePipeMessages.UNSUPPORTED_OUTPUT_DATATYPE_FMT, outPutDataType)); + } + + public int getIntValue() { + return intValue; + } + + public void setIntValue(final int intValue) { + this.intValue = intValue; + } + + public long getLongValue() { + return longValue; + } + + public void setLongValue(final long longValue) { + this.longValue = longValue; + } + + public float getFloatValue() { + return floatValue; + } + + public void setFloatValue(final float floatValue) { + this.floatValue = floatValue; + } + + public double getDoubleValue() { + return doubleValue; + } + + public void setDoubleValue(final double doubleValue) { + this.doubleValue = doubleValue; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java index 2e9fc5ebc40e..673c504442c5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java @@ -19,14 +19,10 @@ package org.apache.iotdb.db.pipe.receiver.transform.converter; -import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; -import org.apache.iotdb.db.i18n.DataNodePipeMessages; -import org.apache.iotdb.db.utils.DataNodeDateTimeUtils; -import org.apache.iotdb.db.utils.TypeInferenceUtils; +import org.apache.iotdb.db.utils.TypeServices; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.external.commons.lang3.StringUtils; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.DateUtils; @@ -35,6 +31,15 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; +import static org.apache.iotdb.db.utils.TypeServices.DEFAULT_DATE; +import static org.apache.iotdb.db.utils.TypeServices.parseBlob; +import static org.apache.iotdb.db.utils.TypeServices.parseDate; +import static org.apache.iotdb.db.utils.TypeServices.parseDouble; +import static org.apache.iotdb.db.utils.TypeServices.parseFloat; +import static org.apache.iotdb.db.utils.TypeServices.parseString; +import static org.apache.iotdb.db.utils.TypeServices.parseText; +import static org.apache.iotdb.db.utils.TypeServices.parseTimestamp; + public class ValueConverter { @FunctionalInterface @@ -308,8 +313,6 @@ public static Object convert( private static final int TRUE_DATE = DateUtils.parseDateExpressionToInt(LocalDate.of(1970, 1, 2)); private static final int FALSE_DATE = DateUtils.parseDateExpressionToInt(LocalDate.of(1970, 1, 1)); - private static final int DEFAULT_DATE = - DateUtils.parseDateExpressionToInt(LocalDate.of(1970, 1, 1)); public static int convertBooleanToInt32(final boolean value) { return value ? 1 : 0; @@ -730,125 +733,9 @@ public static Object parse(final String value, final TSDataType dataType) { if (value == null) { return null; } - switch (dataType) { - case BOOLEAN: - return Boolean.parseBoolean(value); - case INT32: - return parseInteger(value); - case INT64: - return parseLong(value); - case FLOAT: - return parseFloat(value); - case DOUBLE: - return parseDouble(value); - case TEXT: - return parseText(value); - case TIMESTAMP: - return parseTimestamp(value); - case DATE: - return parseDate(value); - case BLOB: - return parseBlob(value); - case STRING: - return parseString(value); - default: - throw new UnsupportedOperationException( - DataNodePipeMessages.UNSUPPORTED_DATA_TYPE + dataType); - } - } - - private static Binary parseBlob(final String value) { - return new Binary(value, TSFileConfig.STRING_CHARSET); - } - - private static int parseInteger(final String value) { - try { - return Integer.parseInt(value); - } catch (Exception e) { - return 0; - } - } - - private static long parseLong(final String value) { - try { - return Long.parseLong(value); - } catch (Exception e) { - return 0L; - } - } - - private static float parseFloat(final String value) { - try { - return Float.parseFloat(value); - } catch (Exception e) { - return 0.0f; - } - } - - private static double parseDouble(final String value) { - try { - return Double.parseDouble(value); - } catch (Exception e) { - return 0.0d; - } - } - - private static long parseTimestamp(final String value) { - if (value == null || value.isEmpty()) { - return 0L; - } - try { - return TypeInferenceUtils.isNumber(value) - ? Long.parseLong(value) - : DataNodeDateTimeUtils.parseDateTimeExpressionToLong( - StringUtils.trim(value), ZoneOffset.UTC); - } catch (final Exception e) { - return 0L; - } - } - - private static int parseDate(final String value) { - if (value == null) { - return DEFAULT_DATE; - } - final String trimmedValue = StringUtils.trim(value); - if (trimmedValue.isEmpty()) { - return DEFAULT_DATE; - } - if (TypeInferenceUtils.isNumber(trimmedValue)) { - try { - int date = Integer.parseInt(trimmedValue); - DateUtils.parseIntToLocalDate(date); - return date; - } catch (final Exception e) { - return DEFAULT_DATE; - } - } - try { - return DateTimeUtils.parseDateExpressionToInt(trimmedValue); - } catch (final Exception e) { - return parseDateTimeToDate(trimmedValue); - } - } - - private static int parseDateTimeToDate(final String value) { - try { - return DateUtils.parseDateExpressionToInt( - Instant.ofEpochMilli( - DateTimeUtils.convertDatetimeStrToLong(value, ZoneOffset.UTC, 0, "ms")) - .atZone(ZoneOffset.UTC) - .toLocalDate()); - } catch (final Exception e) { - return DEFAULT_DATE; - } - } - - private static Binary parseString(final String value) { - return new Binary(value, TSFileConfig.STRING_CHARSET); - } - - private static Binary parseText(final String value) { - return new Binary(value, TSFileConfig.STRING_CHARSET); + return TypeServices.ValueConversion.VALUE_PARSER_NO_EXCEPTION_SERVICE + .call(Type.fromTsDataType(dataType)) + .apply(value); } private ValueConverter() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java index 394d67fd6be1..39cde6ea4db8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java @@ -39,11 +39,14 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.encoding.encoder.TSEncodingBuilder; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.RamUsageEstimator; @@ -137,16 +140,8 @@ public class InsertNodeMemoryEstimator { // ============================= Primitive Type Wrapper Classes ========= - private static final long SIZE_OF_LONG = - RamUsageEstimator.alignObjectSize(Long.BYTES + NUM_BYTES_OBJECT_HEADER); private static final long SIZE_OF_INT = RamUsageEstimator.alignObjectSize(Integer.BYTES + NUM_BYTES_OBJECT_HEADER); - private static final long SIZE_OF_DOUBLE = - RamUsageEstimator.alignObjectSize(Double.BYTES + NUM_BYTES_OBJECT_HEADER); - private static final long SIZE_OF_FLOAT = - RamUsageEstimator.alignObjectSize(Float.BYTES + NUM_BYTES_OBJECT_HEADER); - private static final long SIZE_OF_BOOLEAN = - RamUsageEstimator.alignObjectSize(1 + NUM_BYTES_OBJECT_HEADER); private static final long SIZE_OF_STRING = RamUsageEstimator.shallowSizeOfInstance(String.class); private static final long SIZE_OF_ARRAYLIST = @@ -569,10 +564,6 @@ private static long sizeOfProgressIndex( // =============================Write================================== - private static long sizeOfBinary(final Binary binary) { - return Objects.nonNull(binary) ? binary.ramBytesUsed() : 0L; - } - public static long sizeOfColumns( final Object[] columns, final MeasurementSchema[] measurementSchemas) { if (Objects.isNull(columns)) { @@ -595,42 +586,9 @@ public static long sizeOfColumns( || measurementSchemas[i].getType() == null) { continue; } - switch (measurementSchemas[i].getType()) { - case INT64: - case TIMESTAMP: - { - size += RamUsageEstimator.sizeOf((long[]) columns[i]); - break; - } - case DATE: - case INT32: - { - size += RamUsageEstimator.sizeOf((int[]) columns[i]); - break; - } - case DOUBLE: - { - size += RamUsageEstimator.sizeOf((double[]) columns[i]); - break; - } - case FLOAT: - { - size += RamUsageEstimator.sizeOf((float[]) columns[i]); - break; - } - case BOOLEAN: - { - size += RamUsageEstimator.sizeOf((boolean[]) columns[i]); - break; - } - case STRING: - case TEXT: - case BLOB: - case OBJECT: - { - size += RamUsageEstimator.sizeOf((Binary[]) columns[i]); - break; - } + final TSDataType dataType = measurementSchemas[i].getType(); + if (dataType != TSDataType.UNKNOWN && dataType != TSDataType.VECTOR) { + size += Type.fromTsDataType(dataType).estimateArraySize(columns[i]); } } return size; @@ -664,42 +622,11 @@ public static long sizeOfValues( size += NUM_BYTES_OBJECT_HEADER; continue; } - switch (measurementSchemas[i].getType()) { - case INT64: - case TIMESTAMP: - { - size += SIZE_OF_LONG; - break; - } - case DATE: - case INT32: - { - size += SIZE_OF_INT; - break; - } - case DOUBLE: - { - size += SIZE_OF_DOUBLE; - break; - } - case FLOAT: - { - size += SIZE_OF_FLOAT; - break; - } - case BOOLEAN: - { - size += SIZE_OF_BOOLEAN; - break; - } - case STRING: - case TEXT: - case BLOB: - { - final Binary binary = (Binary) values[i]; - size += sizeOfBinary(binary); - } - } + final TSDataType dataType = measurementSchemas[i].getType(); + size += + TypeServices.Memory.INSERT_NODE_VALUE_SIZE_SERVICE + .call(Type.fromTsDataType(dataType)) + .applyAsLong(values[i]); } return size; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java index 33ceacf278cd..ebdc349c120c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java @@ -36,6 +36,7 @@ import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; @@ -115,7 +116,8 @@ public static Pair calculateTabletRowCountAndMemory(RowRecord totalSizeInBytes += binary == null ? 8 : binary.ramBytesUsed(); } else { totalSizeInBytes += - roundUpToMultiple(TsPrimitiveType.getByType(tsDataType).getSize() + 8, 8); + roundUpToMultiple( + Type.fromTsDataType(tsDataType).getTsPrimitiveType().getSize() + 8, 8); } } } @@ -163,7 +165,8 @@ public static Pair calculateTabletRowCountAndMemory(BatchData final Binary binary = batchData.getBinary(); totalSizeInBytes += binary == null ? 8 : binary.ramBytesUsed(); } else { - totalSizeInBytes += roundUpToMultiple(TsPrimitiveType.getByType(type).getSize() + 8, 8); + totalSizeInBytes += + roundUpToMultiple(Type.fromTsDataType(type).getTsPrimitiveType().getSize() + 8, 8); } } } @@ -279,7 +282,8 @@ public static long calculateBatchDataRamBytesUsed(BatchData batchData) { totalSizeInBytes += roundUpToMultiple((binary == null ? 8 : binary.getLength() + 8) + 8, 8); } else { - totalSizeInBytes += roundUpToMultiple(TsPrimitiveType.getByType(type).getSize() + 8, 8); + totalSizeInBytes += + roundUpToMultiple(Type.fromTsDataType(type).getTsPrimitiveType().getSize() + 8, 8); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java index cb367a194945..82cacc75c439 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.pipe.sink.payload.evolvable.batch; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; -import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; @@ -30,19 +29,17 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -360,50 +357,8 @@ public static Tablet copyTablet(final Tablet tablet) { private static Object copyValueList( final Object valueList, final TSDataType dataType, final int rowSize) { - switch (dataType) { - case BOOLEAN: - final boolean[] boolValues = (boolean[]) valueList; - final boolean[] copiedBoolValues = new boolean[rowSize]; - System.arraycopy(boolValues, 0, copiedBoolValues, 0, rowSize); - return copiedBoolValues; - case INT32: - final int[] intValues = (int[]) valueList; - final int[] copiedIntValues = new int[rowSize]; - System.arraycopy(intValues, 0, copiedIntValues, 0, rowSize); - return copiedIntValues; - case DATE: - final LocalDate[] dateValues = (LocalDate[]) valueList; - final LocalDate[] copiedDateValues = new LocalDate[rowSize]; - System.arraycopy(dateValues, 0, copiedDateValues, 0, rowSize); - return copiedDateValues; - case INT64: - case TIMESTAMP: - final long[] longValues = (long[]) valueList; - final long[] copiedLongValues = new long[rowSize]; - System.arraycopy(longValues, 0, copiedLongValues, 0, rowSize); - return copiedLongValues; - case FLOAT: - final float[] floatValues = (float[]) valueList; - final float[] copiedFloatValues = new float[rowSize]; - System.arraycopy(floatValues, 0, copiedFloatValues, 0, rowSize); - return copiedFloatValues; - case DOUBLE: - final double[] doubleValues = (double[]) valueList; - final double[] copiedDoubleValues = new double[rowSize]; - System.arraycopy(doubleValues, 0, copiedDoubleValues, 0, rowSize); - return copiedDoubleValues; - case TEXT: - case BLOB: - case STRING: - final Binary[] binaryValues = (Binary[]) valueList; - final Binary[] copiedBinaryValues = new Binary[rowSize]; - System.arraycopy(binaryValues, 0, copiedBinaryValues, 0, rowSize); - return copiedBinaryValues; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodePipeMessages.PIPE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataType)); - } + final Object copiedValueList = Type.fromTsDataType(dataType).createArray(rowSize); + System.arraycopy(valueList, 0, copiedValueList, 0, rowSize); + return copiedValueList; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcda/OpcDaServerHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcda/OpcDaServerHandle.java index 76eebecbb453..8bd84d1db72a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcda/OpcDaServerHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcda/OpcDaServerHandle.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.sink.util.sorter.PipeTreeModelTabletEventSorter; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.pipe.api.exception.PipeException; import com.sun.jna.Memory; @@ -30,28 +31,23 @@ import com.sun.jna.platform.win32.COM.IUnknown; import com.sun.jna.platform.win32.COM.Unknown; import com.sun.jna.platform.win32.Guid; -import com.sun.jna.platform.win32.OaIdl; import com.sun.jna.platform.win32.Ole32; import com.sun.jna.platform.win32.OleAuto; import com.sun.jna.platform.win32.Variant; import com.sun.jna.platform.win32.WTypes; -import com.sun.jna.platform.win32.WinDef; import com.sun.jna.platform.win32.WinError; import com.sun.jna.platform.win32.WinNT; import com.sun.jna.ptr.IntByReference; import com.sun.jna.ptr.PointerByReference; import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Closeable; -import java.sql.Date; -import java.time.LocalDate; import java.util.List; import java.util.Map; import java.util.Objects; @@ -309,69 +305,18 @@ private void writeData(final String itemId, final Variant.VARIANT value) { } private short convertTsDataType2VariantType(final TSDataType dataType) { - switch (dataType) { - case BOOLEAN: - return Variant.VT_BOOL; - case INT32: - return Variant.VT_I4; - case INT64: - return Variant.VT_I8; - case DATE: - case TIMESTAMP: - return Variant.VT_DATE; - case FLOAT: - return Variant.VT_R4; - case DOUBLE: - return Variant.VT_R8; - case TEXT: - case STRING: - // Note that "Variant" does not support "VT_BLOB" data, and not all the DA server - // support this, thus we use "VT_BSTR" to substitute - case BLOB: - case OBJECT: - return Variant.VT_BSTR; - default: - throw new UnSupportedDataTypeException( - DataNodePipeMessages.UNSUPPORTED_DATATYPE + dataType); - } + return TypeServices.Pipe.OPC_DA_VARIANT_TYPE_SERVICE.call(Type.fromTsDataType(dataType)); } private Variant.VARIANT getTabletObjectValue4Opc( final Object column, final int rowIndex, final TSDataType type) { final Variant.VARIANT value = new Variant.VARIANT(); - switch (type) { - case BOOLEAN: - value.setValue(Variant.VT_BOOL, new OaIdl.VARIANT_BOOL(((boolean[]) column)[rowIndex])); - break; - case INT32: - value.setValue(Variant.VT_I4, new WinDef.LONG(((int[]) column)[rowIndex])); - break; - case DATE: - value.setValue( - Variant.VT_DATE, new OaIdl.DATE((Date.valueOf(((LocalDate[]) column)[rowIndex])))); - break; - case INT64: - value.setValue(Variant.VT_I8, new WinDef.LONGLONG(((long[]) column)[rowIndex])); - break; - case TIMESTAMP: - value.setValue( - Variant.VT_DATE, new OaIdl.DATE(new java.util.Date(((long[]) column)[rowIndex]))); - break; - case FLOAT: - value.setValue(Variant.VT_R4, ((float[]) column)[rowIndex]); - break; - case DOUBLE: - value.setValue(Variant.VT_R8, ((double[]) column)[rowIndex]); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - bstr = OleAuto.INSTANCE.SysAllocString(((Binary[]) column)[rowIndex].toString()); - value.setValue(Variant.VT_BSTR, bstr); - break; - default: - throw new UnSupportedDataTypeException(DataNodePipeMessages.UNSUPPORTED_DATATYPE + type); + final WTypes.BSTR allocatedBstr = + TypeServices.Pipe.OPC_DA_TABLET_VALUE_SETTER_SERVICE + .call(Type.fromTsDataType(type)) + .set(value, column, rowIndex); + if (Objects.nonNull(allocatedBstr)) { + bstr = allocatedBstr; } return value; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaNameSpace.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaNameSpace.java index 421ba27f77f0..be57be869711 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaNameSpace.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaNameSpace.java @@ -22,12 +22,12 @@ import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; -import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.sink.protocol.opcua.OpcUaSink; import org.apache.iotdb.db.pipe.sink.util.sorter.PipeTableModelTabletEventSorter; import org.apache.iotdb.db.pipe.sink.util.sorter.PipeTreeModelTabletEventSorter; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.pipe.api.event.Event; import org.apache.tsfile.common.constant.TsFileConstant; @@ -35,8 +35,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; @@ -67,12 +66,9 @@ import org.slf4j.LoggerFactory; import java.nio.file.Paths; -import java.time.LocalDate; -import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Date; import java.util.List; import java.util.Objects; import java.util.Set; @@ -82,6 +78,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.stream.Collectors; public class OpcUaNameSpace extends ManagedNamespaceWithLifecycle { @@ -200,17 +197,9 @@ public static void transferLastValues( private static Object getObjectValue4Opc( final TimeValuePair timeValuePair, final TSDataType dataType) { - final Object value = timeValuePair.getValue().getValue(); - return switch (dataType) { - case DATE -> - new DateTime(new Date(DateUtils.parseIntToDate(((Number) value).intValue()).getTime())); - case TIMESTAMP -> new DateTime(timestampToUtc(((Number) value).longValue())); - case TEXT, BLOB, STRING -> value instanceof Binary ? value.toString() : String.valueOf(value); - case BOOLEAN, INT32, INT64, FLOAT, DOUBLE -> value; - case VECTOR, OBJECT, UNKNOWN -> - throw new UnSupportedDataTypeException( - DataNodePipeMessages.UNSUPPORTED_DATATYPE + dataType); - }; + return TypeServices.Pipe.OPC_UA_LAST_VALUE_CONVERTER_SERVICE + .call(Type.fromTsDataType(dataType)) + .apply(timeValuePair.getValue().getValue()); } public static void transferTabletForClientServerModel( @@ -470,30 +459,15 @@ private UaVariableNode addNode( private static Object getTabletObjectValue4Opc( final Object column, final int rowIndex, final TSDataType type) { - switch (type) { - case BOOLEAN: - return ((boolean[]) column)[rowIndex]; - case INT32: - return ((int[]) column)[rowIndex]; - case DATE: - return new DateTime( - Date.from( - ((LocalDate[]) column)[rowIndex].atStartOfDay(ZoneId.systemDefault()).toInstant())); - case INT64: - return ((long[]) column)[rowIndex]; - case TIMESTAMP: - return new DateTime(timestampToUtc(((long[]) column)[rowIndex])); - case FLOAT: - return ((float[]) column)[rowIndex]; - case DOUBLE: - return ((double[]) column)[rowIndex]; - case TEXT: - case BLOB: - case STRING: - return ((Binary[]) column)[rowIndex].toString(); - default: - throw new UnSupportedDataTypeException(DataNodePipeMessages.UNSUPPORTED_DATATYPE + type); + final Type valueType; + try { + valueType = Type.fromTsDataType(type); + } catch (final UnsupportedOperationException ignored) { + throw new UnSupportedDataTypeException(DataNodePipeMessages.UNSUPPORTED_DATATYPE + type); } + return TypeServices.Pipe.OPC_UA_TABLET_OBJECT_VALUE_GETTER_SERVICE + .call(valueType) + .get(column, rowIndex); } public static long timestampToUtc(final long timeStamp) { @@ -535,6 +509,8 @@ private void transferTabletForPubSubModel( continue; } final TSDataType dataType = tablet.getSchemas().get(columnIndex).getType(); + final Function valueStringifier = + TypeServices.Pipe.OPC_UA_VALUE_STRINGIFIER_SERVICE.call(Type.fromTsDataType(dataType)); // Source name --> Sensor path, like root.test.d_0.s_0 if (!isTableModel) { @@ -568,58 +544,8 @@ private void transferTabletForPubSubModel( eventNode.setTime(new DateTime(timestampToUtc(tablet.getTimestamp(rowIndex)))); // Message --> Value - switch (dataType) { - case BOOLEAN: - eventNode.setMessage( - LocalizedText.english( - Boolean.toString((boolean) tablet.getValue(rowIndex, columnIndex)))); - break; - case INT32: - eventNode.setMessage( - LocalizedText.english( - Integer.toString((int) tablet.getValue(rowIndex, columnIndex)))); - break; - case DATE: - eventNode.setMessage( - LocalizedText.english( - ((LocalDate) tablet.getValue(rowIndex, columnIndex)) - .atStartOfDay(ZoneId.systemDefault()) - .toString())); - break; - case INT64: - eventNode.setMessage( - LocalizedText.english( - Long.toString((long) tablet.getValue(rowIndex, columnIndex)))); - break; - case TIMESTAMP: - eventNode.setMessage( - LocalizedText.english( - DateTimeUtils.convertLongToDate( - (long) tablet.getValue(rowIndex, columnIndex)))); - break; - case FLOAT: - eventNode.setMessage( - LocalizedText.english( - Float.toString((float) tablet.getValue(rowIndex, columnIndex)))); - break; - case DOUBLE: - eventNode.setMessage( - LocalizedText.english( - Double.toString((double) tablet.getValue(rowIndex, columnIndex)))); - break; - case TEXT: - case BLOB: - case STRING: - eventNode.setMessage( - LocalizedText.english(tablet.getValue(rowIndex, columnIndex).toString())); - break; - case VECTOR: - case UNKNOWN: - default: - throw new PipeRuntimeNonCriticalException( - DataNodePipeMessages.UNSUPPORTED_DATA_TYPE - + tablet.getSchemas().get(columnIndex).getType()); - } + eventNode.setMessage( + LocalizedText.english(valueStringifier.apply(tablet.getValue(rowIndex, columnIndex)))); // Send the event getServer().getEventNotifier().fire(eventNode); @@ -629,31 +555,13 @@ private void transferTabletForPubSubModel( } public static NodeId convertToOpcDataType(final TSDataType type) { - switch (type) { - case BOOLEAN: - return Identifiers.Boolean; - case INT32: - return Identifiers.Int32; - case DATE: - case TIMESTAMP: - return Identifiers.DateTime; - case INT64: - return Identifiers.Int64; - case FLOAT: - return Identifiers.Float; - case DOUBLE: - return Identifiers.Double; - case TEXT: - case BLOB: - case STRING: - return Identifiers.String; - case VECTOR: - case OBJECT: - case UNKNOWN: - default: - throw new PipeRuntimeNonCriticalException( - DataNodePipeMessages.UNSUPPORTED_DATA_TYPE + type); + final Type dataType; + try { + dataType = Type.fromTsDataType(type); + } catch (final UnsupportedOperationException ignored) { + throw new PipeRuntimeNonCriticalException(DataNodePipeMessages.UNSUPPORTED_DATA_TYPE + type); } + return TypeServices.Pipe.OPC_UA_DATA_TYPE_SERVICE.call(dataType).get(); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java index edea5002683e..2dad8191a4ab 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java @@ -28,14 +28,15 @@ import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -502,113 +503,24 @@ private static Pair readValuesFromBufferWithMemory( continue; } - switch (types[i]) { - case BOOLEAN: - final boolean[] boolValues = new boolean[rowSize]; - if (isValueColumnsNotNull) { - for (int index = 0; index < rowSize; index++) { - boolValues[index] = readBooleanByte(byteBuffer, "boolean value"); - } - } - values[i] = boolValues; - // Calculate memory for boolean array: array header + 1 byte per element (aligned) - memorySize += - org.apache.tsfile.utils.RamUsageEstimator.alignObjectSize( - NUM_BYTES_ARRAY_HEADER + rowSize); - break; - case INT32: - case DATE: - final int[] intValues = new int[rowSize]; - if (isValueColumnsNotNull) { - for (int index = 0; index < rowSize; index++) { - intValues[index] = ReadWriteIOUtils.readInt(byteBuffer); - } - } - values[i] = intValues; - // Calculate memory for int array: array header + 4 bytes per element (aligned) - memorySize += - org.apache.tsfile.utils.RamUsageEstimator.alignObjectSize( - NUM_BYTES_ARRAY_HEADER + (long) Integer.BYTES * rowSize); - break; - case INT64: - case TIMESTAMP: - final long[] longValues = new long[rowSize]; - if (isValueColumnsNotNull) { - for (int index = 0; index < rowSize; index++) { - longValues[index] = ReadWriteIOUtils.readLong(byteBuffer); - } - } - values[i] = longValues; - // Calculate memory for long array: array header + 8 bytes per element (aligned) - memorySize += - org.apache.tsfile.utils.RamUsageEstimator.alignObjectSize( - NUM_BYTES_ARRAY_HEADER + (long) Long.BYTES * rowSize); - break; - case FLOAT: - final float[] floatValues = new float[rowSize]; - if (isValueColumnsNotNull) { - for (int index = 0; index < rowSize; index++) { - floatValues[index] = ReadWriteIOUtils.readFloat(byteBuffer); - } - } - values[i] = floatValues; - // Calculate memory for float array: array header + 4 bytes per element (aligned) - memorySize += - org.apache.tsfile.utils.RamUsageEstimator.alignObjectSize( - NUM_BYTES_ARRAY_HEADER + (long) Float.BYTES * rowSize); - break; - case DOUBLE: - final double[] doubleValues = new double[rowSize]; - if (isValueColumnsNotNull) { - for (int index = 0; index < rowSize; index++) { - doubleValues[index] = ReadWriteIOUtils.readDouble(byteBuffer); - } - } - values[i] = doubleValues; - // Calculate memory for double array: array header + 8 bytes per element (aligned) - memorySize += - org.apache.tsfile.utils.RamUsageEstimator.alignObjectSize( - NUM_BYTES_ARRAY_HEADER + (long) Double.BYTES * rowSize); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - // Handle object array type: Binary[] is an array of objects - final Binary[] binaryValues = new Binary[rowSize]; - // Calculate memory for Binary array: array header + object references - long binaryArrayMemory = - org.apache.tsfile.utils.RamUsageEstimator.alignObjectSize( - NUM_BYTES_ARRAY_HEADER + NUM_BYTES_OBJECT_REF * rowSize); - - if (isValueColumnsNotNull) { - for (int index = 0; index < rowSize; index++) { - final boolean isNotNull = readBooleanByte(byteBuffer, "binary value existence"); - if (isNotNull) { - binaryValues[index] = ReadWriteIOUtils.readBinary(byteBuffer); - // Calculate memory for each Binary object during deserialization - binaryArrayMemory += binaryValues[index].ramBytesUsed(); - } else { - binaryValues[index] = Binary.EMPTY_VALUE; - // EMPTY_VALUE also has memory cost - binaryArrayMemory += Binary.EMPTY_VALUE.ramBytesUsed(); - } - } - } else { - Arrays.fill(binaryValues, Binary.EMPTY_VALUE); - // Calculate memory for all EMPTY_VALUE - binaryArrayMemory += (long) rowSize * Binary.EMPTY_VALUE.ramBytesUsed(); - } - values[i] = binaryValues; - // Add calculated Binary array memory to total - memorySize += binaryArrayMemory; - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodePipeMessages - .PIPE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_WHEN_CONVERT_DATA_AT_CLIENT_405429CC, - types[i])); + final Type type = Type.fromTsDataType(types[i]); + final Type serializedArrayType = + types[i] == TSDataType.DATE ? Type.fromTsDataType(TSDataType.INT32) : type; + values[i] = + isValueColumnsNotNull + ? serializedArrayType.deserializeArray(byteBuffer, rowSize) + : TypeServices.StorageEngine.PRIMITIVE_ARRAY_ALLOCATOR_SERVICE + .call(type) + .apply(rowSize); + final boolean isBinaryType = types[i].isBinary(); + if (isBinaryType && !isValueColumnsNotNull) { + Arrays.fill((Binary[]) values[i], Binary.EMPTY_VALUE); + } + memorySize += type.estimateArraySize(values[i]); + if (isBinaryType) { + for (final Binary binary : (Binary[]) values[i]) { + memorySize += binary.ramBytesUsed(); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/PipeInsertEventSorter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/PipeInsertEventSorter.java index db81b83a7d61..73c35b8cc7ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/PipeInsertEventSorter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/PipeInsertEventSorter.java @@ -19,17 +19,16 @@ package org.apache.iotdb.db.pipe.sink.util.sorter; -import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; -import java.time.LocalDate; +import java.lang.reflect.Array; import java.util.Objects; public class PipeInsertEventSorter { @@ -122,86 +121,27 @@ protected Object reorderValueListAndBitMap( if (Objects.isNull(valueList)) { return null; } - switch (dataType) { - case BOOLEAN: - final boolean[] boolValues = (boolean[]) valueList; - final boolean[] deDuplicatedBoolValues = new boolean[boolValues.length]; - for (int i = 0; i < deDuplicatedSize; i++) { - deDuplicatedBoolValues[i] = - boolValues[getLastNonnullIndex(i, originalBitMap, deDuplicatedBitMap)]; - } - return deDuplicatedBoolValues; - case INT32: - final int[] intValues = (int[]) valueList; - final int[] deDuplicatedIntValues = new int[intValues.length]; - for (int i = 0; i < deDuplicatedSize; i++) { - deDuplicatedIntValues[i] = - intValues[getLastNonnullIndex(i, originalBitMap, deDuplicatedBitMap)]; - } - return deDuplicatedIntValues; - case DATE: - // DATE type: Tablet uses LocalDate[], InsertTabletStatement uses int[] - if (dataAdapter.isDateStoredAsLocalDate(columnIndex)) { - // Tablet: LocalDate[] - final LocalDate[] dateValues = (LocalDate[]) valueList; - final LocalDate[] deDuplicatedDateValues = new LocalDate[dateValues.length]; - for (int i = 0; i < deDuplicatedSize; i++) { - deDuplicatedDateValues[i] = - dateValues[getLastNonnullIndex(i, originalBitMap, deDuplicatedBitMap)]; - } - return deDuplicatedDateValues; - } else { - // InsertTabletStatement: int[] - final int[] intDateValues = (int[]) valueList; - final int[] deDuplicatedIntDateValues = new int[intDateValues.length]; - for (int i = 0; i < deDuplicatedSize; i++) { - deDuplicatedIntDateValues[i] = - intDateValues[getLastNonnullIndex(i, originalBitMap, deDuplicatedBitMap)]; - } - return deDuplicatedIntDateValues; - } - case INT64: - case TIMESTAMP: - final long[] longValues = (long[]) valueList; - final long[] deDuplicatedLongValues = new long[longValues.length]; - for (int i = 0; i < deDuplicatedSize; i++) { - deDuplicatedLongValues[i] = - longValues[getLastNonnullIndex(i, originalBitMap, deDuplicatedBitMap)]; - } - return deDuplicatedLongValues; - case FLOAT: - final float[] floatValues = (float[]) valueList; - final float[] deDuplicatedFloatValues = new float[floatValues.length]; - for (int i = 0; i < deDuplicatedSize; i++) { - deDuplicatedFloatValues[i] = - floatValues[getLastNonnullIndex(i, originalBitMap, deDuplicatedBitMap)]; - } - return deDuplicatedFloatValues; - case DOUBLE: - final double[] doubleValues = (double[]) valueList; - final double[] deDuplicatedDoubleValues = new double[doubleValues.length]; - for (int i = 0; i < deDuplicatedSize; i++) { - deDuplicatedDoubleValues[i] = - doubleValues[getLastNonnullIndex(i, originalBitMap, deDuplicatedBitMap)]; - } - return deDuplicatedDoubleValues; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - final Binary[] binaryValues = (Binary[]) valueList; - final Binary[] deDuplicatedBinaryValues = new Binary[binaryValues.length]; - for (int i = 0; i < deDuplicatedSize; i++) { - deDuplicatedBinaryValues[i] = - binaryValues[getLastNonnullIndex(i, originalBitMap, deDuplicatedBitMap)]; - } - return deDuplicatedBinaryValues; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodePipeMessages.PIPE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataType)); + final Type type = + TypeServices.Pipe.PIPE_INSERT_EVENT_VALUE_LIST_TYPE_SERVICE + .call(Type.fromTsDataType(dataType)) + .apply(dataType != TSDataType.DATE || dataAdapter.isDateStoredAsLocalDate(columnIndex)); + return reorderValueList(valueList, type, originalBitMap, deDuplicatedBitMap); + } + + private Object reorderValueList( + final Object valueList, + final Type type, + final BitMap originalBitMap, + final BitMap deDuplicatedBitMap) { + final Object deDuplicatedValues = type.createArray(Array.getLength(valueList)); + for (int i = 0; i < deDuplicatedSize; i++) { + type.copyArrayElement( + valueList, + getLastNonnullIndex(i, originalBitMap, deDuplicatedBitMap), + deDuplicatedValues, + i); } + return deDuplicatedValues; } private int getLastNonnullIndex( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index be0a00bddf82..ce0c5d674500 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -48,14 +48,9 @@ import org.apache.iotdb.commons.queryengine.common.SqlDialect; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.plan.planner.plan.parameter.InputLocation; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BinaryLiteral; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Literal; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NullLiteral; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.parser.ParsingException; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.audit.DNAuditLogger; @@ -143,6 +138,7 @@ import org.apache.iotdb.db.utils.QueryDataSetUtils; import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.iotdb.db.utils.SetThreadName; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.rpc.stmt.PreparedParameterSerde; @@ -218,6 +214,7 @@ import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.column.TsBlockSerde; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.TimeFilterApi; import org.apache.tsfile.utils.Binary; @@ -701,34 +698,9 @@ private static Pair convertToLiteralWithString(DeserializedPara return new Pair<>(new NullLiteral(), "NULL"); } - switch (param.type) { - case BOOLEAN: - String boolStr = (Boolean) param.value ? "true" : "false"; - return new Pair<>(new BooleanLiteral(boolStr), boolStr); - case INT32: - case INT64: - String numStr = String.valueOf(param.value); - return new Pair<>(new LongLiteral(numStr), numStr); - case FLOAT: - String floatStr = String.valueOf(param.value); - return new Pair<>(new DoubleLiteral((Float) param.value), floatStr); - case DOUBLE: - String doubleStr = String.valueOf(param.value); - return new Pair<>(new DoubleLiteral((Double) param.value), doubleStr); - case TEXT: - case STRING: - String strVal = (String) param.value; - // Escape single quotes for SQL - String escapedStr = "'" + strVal.replace("'", "''") + "'"; - return new Pair<>(new StringLiteral(strVal), escapedStr); - case BLOB: - byte[] bytes = (byte[]) param.value; - String hexStr = "X'" + PreparedParameterSerde.bytesToHex(bytes) + "'"; - return new Pair<>(new BinaryLiteral(bytes), hexStr); - default: - throw new IllegalArgumentException( - DataNodeMiscMessages.UNKNOWN_PARAMETER_TYPE + param.type); - } + return TypeServices.ValueConversion.PREPARED_PARAMETER_LITERAL_SERVICE + .call(Type.fromTsDataType(param.type)) + .apply(param.value); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java index 6f626a0269c8..ad268f9c43d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java @@ -20,15 +20,9 @@ package org.apache.iotdb.db.queryengine.execution.aggregation; import org.apache.iotdb.calc.execution.aggregation.Accumulator; -import org.apache.iotdb.calc.execution.aggregation.BinaryModeAccumulator; -import org.apache.iotdb.calc.execution.aggregation.BooleanModeAccumulator; import org.apache.iotdb.calc.execution.aggregation.CentralMomentAccumulator; import org.apache.iotdb.calc.execution.aggregation.CorrelationAccumulator; import org.apache.iotdb.calc.execution.aggregation.CovarianceAccumulator; -import org.apache.iotdb.calc.execution.aggregation.DoubleModeAccumulator; -import org.apache.iotdb.calc.execution.aggregation.FloatModeAccumulator; -import org.apache.iotdb.calc.execution.aggregation.IntModeAccumulator; -import org.apache.iotdb.calc.execution.aggregation.LongModeAccumulator; import org.apache.iotdb.calc.execution.aggregation.RegressionAccumulator; import org.apache.iotdb.calc.execution.aggregation.VarianceAccumulator; import org.apache.iotdb.common.rpc.thrift.TAggregationType; @@ -39,12 +33,14 @@ import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import java.util.ArrayList; import java.util.List; import java.util.Map; import static com.google.common.base.Preconditions.checkState; +import static org.apache.iotdb.db.utils.TypeServices.Aggregation.MODE_ACCUMULATOR_PROVIDER_SERVICE; public class AccumulatorFactory { @@ -210,27 +206,13 @@ private static Accumulator createBuiltinSingleInputAccumulator( } private static Accumulator createModeAccumulator(TSDataType tsDataType) { - switch (tsDataType) { - case BOOLEAN: - return new BooleanModeAccumulator(); - case BLOB: - case STRING: - case TEXT: - return new BinaryModeAccumulator(); - case DATE: - case INT32: - return new IntModeAccumulator(); - case TIMESTAMP: - case INT64: - return new LongModeAccumulator(); - case FLOAT: - return new FloatModeAccumulator(); - case DOUBLE: - return new DoubleModeAccumulator(); - case OBJECT: - default: - throw new IllegalArgumentException(DataNodeQueryMessages.UNKNOWN_DATA_TYPE + tsDataType); + final Type type; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException ignored) { + throw new IllegalArgumentException(DataNodeQueryMessages.UNKNOWN_DATA_TYPE + tsDataType); } + return MODE_ACCUMULATOR_PROVIDER_SERVICE.call(type).get(); } @TestOnly diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java index 27da8f412efc..5265a1372b18 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java @@ -45,32 +45,29 @@ public AvgAccumulator(TSDataType seriesDataType) { @Override public void addInput(Column[] columns, BitMap bitMap) { - switch (seriesDataType) { - case INT32: - addIntInput(columns, bitMap); - return; - case INT64: - addLongInput(columns, bitMap); - return; - case FLOAT: - addFloatInput(columns, bitMap); - return; - case DOUBLE: - addDoubleInput(columns, bitMap); - return; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_AGGREGATION_AVG_S_D1DAD6A6, - seriesDataType)); + final boolean isNumeric; + try { + isNumeric = seriesDataType.isNumeric(); + } catch (final UnSupportedDataTypeException ignored) { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, seriesDataType)); + } + if (!isNumeric) { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, seriesDataType)); + } + final int count = columns[0].getPositionCount(); + for (int i = 0; i < count; i++) { + if (bitMap != null && !bitMap.isMarked(i)) { + continue; + } + if (!columns[1].isNull(i)) { + initResult = true; + countValue++; + sumValue += columns[1].getDouble(i); + } } } @@ -184,60 +181,4 @@ public TSDataType getFinalType() { public int getPartialResultSize() { return 2; } - - private void addIntInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - initResult = true; - countValue++; - sumValue += column[1].getInt(i); - } - } - } - - private void addLongInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - initResult = true; - countValue++; - sumValue += column[1].getLong(i); - } - } - } - - private void addFloatInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - initResult = true; - countValue++; - sumValue += column[1].getFloat(i); - } - } - } - - private void addDoubleInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - initResult = true; - countValue++; - sumValue += column[1].getDouble(i); - } - } - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java index 597e09d98197..554a9eb3322c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java @@ -21,56 +21,46 @@ import org.apache.iotdb.calc.execution.aggregation.Accumulator; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.utils.TypeServices; +import org.apache.iotdb.db.utils.TypeServices.Aggregation.ExtremeValueAccumulatorStrategy; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.util.EnumSet; +import java.util.Set; + import static com.google.common.base.Preconditions.checkArgument; -public class ExtremeAccumulator implements Accumulator { +public class ExtremeAccumulator + implements Accumulator, TypeServices.Aggregation.ExtremeValueAccumulator { + + private static final Set SUPPORTED_TYPES = + EnumSet.of(TSDataType.INT32, TSDataType.INT64, TSDataType.FLOAT, TSDataType.DOUBLE); private final TSDataType seriesDataType; private final TsPrimitiveType extremeResult; + private final ExtremeValueAccumulatorStrategy strategy; private boolean initResult; public ExtremeAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - this.extremeResult = TsPrimitiveType.getByType(seriesDataType); + final Type type = Type.fromTsDataType(seriesDataType); + this.extremeResult = type.getTsPrimitiveType(); + this.strategy = TypeServices.Aggregation.EXTREME_VALUE_ACCUMULATOR_STRATEGY_SERVICE.call(type); + ensureSupported(); } @Override public void addInput(Column[] columns, BitMap bitMap) { - switch (seriesDataType) { - case INT32: - addIntInput(columns, bitMap); - return; - case INT64: - addLongInput(columns, bitMap); - return; - case FLOAT: - addFloatInput(columns, bitMap); - return; - case DOUBLE: - addDoubleInput(columns, bitMap); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EXTREME_S_84B651D3, - seriesDataType)); - } + strategy.addInput(this, columns, bitMap); } // partialResult should be like: | PartialExtremeValue | @@ -82,32 +72,7 @@ public void addIntermediate(Column[] partialResult) { if (partialResult[0].isNull(0)) { return; } - switch (seriesDataType) { - case INT32: - updateIntResult(partialResult[0].getInt(0)); - break; - case INT64: - updateLongResult(partialResult[0].getLong(0)); - break; - case FLOAT: - updateFloatResult(partialResult[0].getFloat(0)); - break; - case DOUBLE: - updateDoubleResult(partialResult[0].getDouble(0)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EXTREME_S_84B651D3, - seriesDataType)); - } + strategy.addIntermediate(this, partialResult[0]); } @Override @@ -115,36 +80,8 @@ public void addStatistics(Statistics statistics) { if (statistics == null) { return; } - switch (seriesDataType) { - case INT32: - updateIntResult(((Number) statistics.getMaxValue()).intValue()); - updateIntResult(((Number) statistics.getMinValue()).intValue()); - break; - case INT64: - updateLongResult(((Number) statistics.getMaxValue()).longValue()); - updateLongResult(((Number) statistics.getMinValue()).longValue()); - break; - case FLOAT: - updateFloatResult(((Number) statistics.getMaxValue()).floatValue()); - updateFloatResult(((Number) statistics.getMinValue()).floatValue()); - break; - case DOUBLE: - updateDoubleResult(((Number) statistics.getMaxValue()).doubleValue()); - updateDoubleResult(((Number) statistics.getMinValue()).doubleValue()); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EXTREME_S_84B651D3, - seriesDataType)); - } + strategy.addStatistics(this, statistics.getMaxValue()); + strategy.addStatistics(this, statistics.getMinValue()); } @Override @@ -153,32 +90,7 @@ public void setFinal(Column finalResult) { return; } initResult = true; - switch (seriesDataType) { - case INT32: - extremeResult.setInt(finalResult.getInt(0)); - break; - case INT64: - extremeResult.setLong(finalResult.getLong(0)); - break; - case FLOAT: - extremeResult.setFloat(finalResult.getFloat(0)); - break; - case DOUBLE: - extremeResult.setDouble(finalResult.getDouble(0)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EXTREME_S_84B651D3, - seriesDataType)); - } + strategy.setFinal(this, finalResult); } // columnBuilder should be single in ExtremeAccumulator @@ -191,32 +103,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { columnBuilders[0].appendNull(); return; } - switch (seriesDataType) { - case INT32: - columnBuilders[0].writeInt(extremeResult.getInt()); - break; - case INT64: - columnBuilders[0].writeLong(extremeResult.getLong()); - break; - case FLOAT: - columnBuilders[0].writeFloat(extremeResult.getFloat()); - break; - case DOUBLE: - columnBuilders[0].writeDouble(extremeResult.getDouble()); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EXTREME_S_84B651D3, - seriesDataType)); - } + strategy.writeResult(columnBuilders[0], extremeResult); } @Override @@ -225,32 +112,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (seriesDataType) { - case INT32: - columnBuilder.writeInt(extremeResult.getInt()); - break; - case INT64: - columnBuilder.writeLong(extremeResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(extremeResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(extremeResult.getDouble()); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case BOOLEAN: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EXTREME_S_84B651D3, - seriesDataType)); - } + strategy.writeResult(columnBuilder, extremeResult); } @Override @@ -274,19 +136,13 @@ public TSDataType getFinalType() { return extremeResult.getDataType(); } - private void addIntInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateIntResult(column[1].getInt(i)); - } - } + @Override + public TsPrimitiveType getResult() { + return extremeResult; } - private void updateIntResult(int extVal) { + @Override + public void updateIntResult(final int extVal) { int candidateResult = extremeResult.getInt(); if (!initResult || compareExtreme(extVal, candidateResult) > 0) { @@ -295,19 +151,8 @@ private void updateIntResult(int extVal) { } } - private void addLongInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateLongResult(column[1].getLong(i)); - } - } - } - - private void updateLongResult(long extVal) { + @Override + public void updateLongResult(final long extVal) { long candidateResult = extremeResult.getLong(); if (!initResult || compareExtreme(extVal, candidateResult) > 0) { @@ -316,19 +161,8 @@ private void updateLongResult(long extVal) { } } - private void addFloatInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateFloatResult(column[1].getFloat(i)); - } - } - } - - private void updateFloatResult(float extVal) { + @Override + public void updateFloatResult(final float extVal) { float absExtVal = Math.abs(extVal); float candidateResult = extremeResult.getFloat(); float absCandidateResult = Math.abs(extremeResult.getFloat()); @@ -341,19 +175,8 @@ private void updateFloatResult(float extVal) { } } - private void addDoubleInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateDoubleResult(column[1].getDouble(i)); - } - } - } - - private void updateDoubleResult(double extVal) { + @Override + public void updateDoubleResult(final double extVal) { double absExtVal = Math.abs(extVal); double candidateResult = extremeResult.getDouble(); double absCandidateResult = Math.abs(extremeResult.getDouble()); @@ -366,6 +189,23 @@ private void updateDoubleResult(double extVal) { } } + @Override + public void updateBinaryResult(final Binary value) { + throw unsupportedDataTypeException(); + } + + private void ensureSupported() { + if (!SUPPORTED_TYPES.contains(seriesDataType) || !strategy.isSupported()) { + throw unsupportedDataTypeException(); + } + } + + private UnSupportedDataTypeException unsupportedDataTypeException() { + return new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, seriesDataType)); + } + private int compareExtreme(int left, int right) { int absComparison = Long.compare(Math.abs((long) left), Math.abs((long) right)); return absComparison == 0 ? Integer.compare(left, right) : absComparison; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java index 899a33d43c1f..8ffb059d9a90 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java @@ -21,67 +21,50 @@ import org.apache.iotdb.calc.execution.aggregation.Accumulator; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.utils.TypeServices; +import org.apache.iotdb.db.utils.TypeServices.Aggregation.TimeValueAccumulatorStrategy; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.statistics.DateStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; -import java.nio.charset.StandardCharsets; - import static com.google.common.base.Preconditions.checkArgument; -public class FirstValueAccumulator implements Accumulator { +public class FirstValueAccumulator + implements Accumulator, TypeServices.Aggregation.TimeValueAccumulator { protected final TSDataType seriesDataType; protected boolean hasCandidateResult; protected TsPrimitiveType firstValue; + private final TimeValueAccumulatorStrategy strategy; protected long minTime = Long.MAX_VALUE; public FirstValueAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - firstValue = TsPrimitiveType.getByType(seriesDataType); + final Type type = Type.fromTsDataType(seriesDataType); + firstValue = type.getTsPrimitiveType(); + strategy = TypeServices.Aggregation.TIME_VALUE_ACCUMULATOR_STRATEGY_SERVICE.call(type); + ensureSupported(); } // Column should be like: | Time | Value | @Override public void addInput(Column[] columns, BitMap bitMap) { - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(columns, bitMap); - return; - case INT64: - case TIMESTAMP: - addLongInput(columns, bitMap); - return; - case FLOAT: - addFloatInput(columns, bitMap); - return; - case DOUBLE: - addDoubleInput(columns, bitMap); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(columns, bitMap); - return; - case BOOLEAN: - addBooleanInput(columns, bitMap); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_FIRSTVALUE_S_97025F25, - seriesDataType)); - } + addInputWithStrategy(columns, bitMap, true, true); + } + + protected final void addInputWithStrategy( + final Column[] columns, + final BitMap bitMap, + final boolean stopAfterFirstValue, + final boolean bitmapMarksSelected) { + strategy.addInput(this, columns, bitMap, stopAfterFirstValue, bitmapMarksSelected); } // partialResult should be like: | FirstValue | MinTime | @@ -93,37 +76,7 @@ public void addIntermediate(Column[] partialResult) { if (partialResult[0].isNull(0)) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntFirstValue(partialResult[0].getInt(0), partialResult[1].getLong(0)); - break; - case INT64: - case TIMESTAMP: - updateLongFirstValue(partialResult[0].getLong(0), partialResult[1].getLong(0)); - break; - case FLOAT: - updateFloatFirstValue(partialResult[0].getFloat(0), partialResult[1].getLong(0)); - break; - case DOUBLE: - updateDoubleFirstValue(partialResult[0].getDouble(0), partialResult[1].getLong(0)); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - updateBinaryFirstValue(partialResult[0].getBinary(0), partialResult[1].getLong(0)); - break; - case BOOLEAN: - updateBooleanFirstValue(partialResult[0].getBoolean(0), partialResult[1].getLong(0)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_FIRSTVALUE_S_97025F25, - seriesDataType)); - } + strategy.addIntermediate(this, partialResult[0], partialResult[1]); } @Override @@ -131,55 +84,7 @@ public void addStatistics(Statistics statistics) { if (statistics == null) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntFirstValue( - ((Number) statistics.getFirstValue()).intValue(), statistics.getStartTime()); - break; - case INT64: - case TIMESTAMP: - updateLongFirstValue( - ((Number) statistics.getFirstValue()).longValue(), statistics.getStartTime()); - break; - case FLOAT: - updateFloatFirstValue( - ((Number) statistics.getFirstValue()).floatValue(), statistics.getStartTime()); - break; - case DOUBLE: - updateDoubleFirstValue( - ((Number) statistics.getFirstValue()).doubleValue(), statistics.getStartTime()); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - if (statistics instanceof DateStatistics) { - updateBinaryFirstValue( - new Binary( - TSDataType.getDateStringValue((Integer) statistics.getFirstValue()), - StandardCharsets.UTF_8), - statistics.getStartTime()); - } else { - if (statistics.getFirstValue() instanceof Binary) { - updateBinaryFirstValue((Binary) statistics.getFirstValue(), statistics.getStartTime()); - } else { - updateBinaryFirstValue( - new Binary(String.valueOf(statistics.getFirstValue()), StandardCharsets.UTF_8), - statistics.getStartTime()); - } - } - break; - case BOOLEAN: - updateBooleanFirstValue((boolean) statistics.getFirstValue(), statistics.getStartTime()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_FIRSTVALUE_S_97025F25, - seriesDataType)); - } + strategy.addStatistics(this, statistics, true); } // finalResult should be single column, like: | finalFirstValue | @@ -188,37 +93,7 @@ public void setFinal(Column finalResult) { reset(); if (!finalResult.isNull(0)) { hasCandidateResult = true; - switch (seriesDataType) { - case INT32: - case DATE: - firstValue.setInt(finalResult.getInt(0)); - break; - case INT64: - case TIMESTAMP: - firstValue.setLong(finalResult.getLong(0)); - break; - case FLOAT: - firstValue.setFloat(finalResult.getFloat(0)); - break; - case DOUBLE: - firstValue.setDouble(finalResult.getDouble(0)); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - firstValue.setBinary(finalResult.getBinary(0)); - break; - case BOOLEAN: - firstValue.setBoolean(finalResult.getBoolean(0)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_FIRSTVALUE_S_97025F25, - seriesDataType)); - } + strategy.setFinal(this, finalResult); } } @@ -233,36 +108,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { columnBuilders[1].appendNull(); return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilders[0].writeInt(firstValue.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilders[0].writeLong(firstValue.getLong()); - break; - case FLOAT: - columnBuilders[0].writeFloat(firstValue.getFloat()); - break; - case DOUBLE: - columnBuilders[0].writeDouble(firstValue.getDouble()); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - columnBuilders[0].writeBinary(firstValue.getBinary()); - break; - case BOOLEAN: - columnBuilders[0].writeBoolean(firstValue.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EXTREME_S_84B651D3, - seriesDataType)); - } + strategy.writeResult(columnBuilders[0], firstValue); columnBuilders[1].writeLong(minTime); } @@ -272,36 +118,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(firstValue.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(firstValue.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(firstValue.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(firstValue.getDouble()); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - columnBuilder.writeBinary(firstValue.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(firstValue.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EXTREME_S_84B651D3, - seriesDataType)); - } + strategy.writeResult(columnBuilder, firstValue); } @Override @@ -331,20 +148,13 @@ public int getPartialResultSize() { return 2; } - protected void addIntInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateIntFirstValue(column[1].getInt(i), column[0].getLong(i)); - return; - } - } + @Override + public TsPrimitiveType getTimeValueResult() { + return firstValue; } - protected void updateIntFirstValue(int value, long curTime) { + @Override + public void updateIntResult(final int value, final long curTime) { hasCandidateResult = true; if (curTime < minTime) { minTime = curTime; @@ -352,20 +162,8 @@ protected void updateIntFirstValue(int value, long curTime) { } } - protected void addLongInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateLongFirstValue(column[1].getLong(i), column[0].getLong(i)); - return; - } - } - } - - protected void updateLongFirstValue(long value, long curTime) { + @Override + public void updateLongResult(final long value, final long curTime) { hasCandidateResult = true; if (curTime < minTime) { minTime = curTime; @@ -373,20 +171,8 @@ protected void updateLongFirstValue(long value, long curTime) { } } - protected void addFloatInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateFloatFirstValue(column[1].getFloat(i), column[0].getLong(i)); - return; - } - } - } - - protected void updateFloatFirstValue(float value, long curTime) { + @Override + public void updateFloatResult(final float value, final long curTime) { hasCandidateResult = true; if (curTime < minTime) { minTime = curTime; @@ -394,20 +180,8 @@ protected void updateFloatFirstValue(float value, long curTime) { } } - protected void addDoubleInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateDoubleFirstValue(column[1].getDouble(i), column[0].getLong(i)); - return; - } - } - } - - protected void updateDoubleFirstValue(double value, long curTime) { + @Override + public void updateDoubleResult(final double value, final long curTime) { hasCandidateResult = true; if (curTime < minTime) { minTime = curTime; @@ -415,20 +189,8 @@ protected void updateDoubleFirstValue(double value, long curTime) { } } - protected void addBooleanInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateBooleanFirstValue(column[1].getBoolean(i), column[0].getLong(i)); - return; - } - } - } - - protected void updateBooleanFirstValue(boolean value, long curTime) { + @Override + public void updateBooleanResult(final boolean value, final long curTime) { hasCandidateResult = true; if (curTime < minTime) { minTime = curTime; @@ -436,24 +198,20 @@ protected void updateBooleanFirstValue(boolean value, long curTime) { } } - protected void addBinaryInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateBinaryFirstValue(column[1].getBinary(i), column[0].getLong(i)); - return; - } - } - } - - protected void updateBinaryFirstValue(Binary value, long curTime) { + @Override + public void updateBinaryResult(final Binary value, final long curTime) { hasCandidateResult = true; if (curTime < minTime) { minTime = curTime; firstValue.setBinary(value); } } + + private void ensureSupported() { + if (!strategy.isSupported()) { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, seriesDataType)); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueDescAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueDescAccumulator.java index 0a249b435141..8e09e7c1861d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueDescAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueDescAccumulator.java @@ -34,82 +34,9 @@ public boolean hasFinalResult() { return false; } - // Don't break in advance @Override - protected void addIntInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateIntFirstValue(column[1].getInt(i), column[0].getLong(i)); - } - } - } - - @Override - protected void addLongInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateLongFirstValue(column[1].getLong(i), column[0].getLong(i)); - } - } - } - - @Override - protected void addFloatInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateFloatFirstValue(column[1].getFloat(i), column[0].getLong(i)); - } - } - } - - @Override - protected void addDoubleInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateDoubleFirstValue(column[1].getDouble(i), column[0].getLong(i)); - } - } - } - - @Override - protected void addBooleanInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateBooleanFirstValue(column[1].getBoolean(i), column[0].getLong(i)); - } - } - } - - @Override - protected void addBinaryInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateBinaryFirstValue(column[1].getBinary(i), column[0].getLong(i)); - } - } + public void addInput(final Column[] columns, final BitMap bitMap) { + // Do not stop after the first value because the input is in descending time order. + addInputWithStrategy(columns, bitMap, false, true); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java index 3c04e6093a62..17b7252e7701 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java @@ -21,66 +21,50 @@ import org.apache.iotdb.calc.execution.aggregation.Accumulator; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.utils.TypeServices; +import org.apache.iotdb.db.utils.TypeServices.Aggregation.TimeValueAccumulatorStrategy; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.statistics.DateStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; -import java.nio.charset.StandardCharsets; - import static com.google.common.base.Preconditions.checkArgument; -public class LastValueAccumulator implements Accumulator { +public class LastValueAccumulator + implements Accumulator, TypeServices.Aggregation.TimeValueAccumulator { protected final TSDataType seriesDataType; protected TsPrimitiveType lastValue; + private final TimeValueAccumulatorStrategy strategy; protected long maxTime = Long.MIN_VALUE; protected boolean initResult = false; public LastValueAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - lastValue = TsPrimitiveType.getByType(seriesDataType); + final Type type = Type.fromTsDataType(seriesDataType); + lastValue = type.getTsPrimitiveType(); + strategy = TypeServices.Aggregation.TIME_VALUE_ACCUMULATOR_STRATEGY_SERVICE.call(type); + ensureSupported(); } // Column should be like: | Time | Value | @Override public void addInput(Column[] columns, BitMap bitMap) { - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(columns, bitMap); - return; - case INT64: - case TIMESTAMP: - addLongInput(columns, bitMap); - return; - case FLOAT: - addFloatInput(columns, bitMap); - return; - case DOUBLE: - addDoubleInput(columns, bitMap); - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - addBinaryInput(columns, bitMap); - return; - case BOOLEAN: - addBooleanInput(columns, bitMap); - return; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_LASTVALUE_S_02ECF8E4, - seriesDataType)); - } + addInputWithStrategy(columns, bitMap, false, true); + } + + protected final void addInputWithStrategy( + final Column[] columns, + final BitMap bitMap, + final boolean stopAfterFirstValue, + final boolean bitmapMarksSelected) { + strategy.addInput(this, columns, bitMap, stopAfterFirstValue, bitmapMarksSelected); } // partialResult should be like: | LastValue | MaxTime | @@ -92,36 +76,7 @@ public void addIntermediate(Column[] partialResult) { if (partialResult[0].isNull(0)) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntLastValue(partialResult[0].getInt(0), partialResult[1].getLong(0)); - break; - case INT64: - case TIMESTAMP: - updateLongLastValue(partialResult[0].getLong(0), partialResult[1].getLong(0)); - break; - case FLOAT: - updateFloatLastValue(partialResult[0].getFloat(0), partialResult[1].getLong(0)); - break; - case DOUBLE: - updateDoubleLastValue(partialResult[0].getDouble(0), partialResult[1].getLong(0)); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - updateBinaryLastValue(partialResult[0].getBinary(0), partialResult[1].getLong(0)); - break; - case BOOLEAN: - updateBooleanLastValue(partialResult[0].getBoolean(0), partialResult[1].getLong(0)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_LASTVALUE_S_02ECF8E4, - seriesDataType)); - } + strategy.addIntermediate(this, partialResult[0], partialResult[1]); } @Override @@ -129,54 +84,7 @@ public void addStatistics(Statistics statistics) { if (statistics == null) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntLastValue( - ((Number) statistics.getLastValue()).intValue(), statistics.getEndTime()); - break; - case INT64: - case TIMESTAMP: - updateLongLastValue( - ((Number) statistics.getLastValue()).longValue(), statistics.getEndTime()); - break; - case FLOAT: - updateFloatLastValue( - ((Number) statistics.getLastValue()).floatValue(), statistics.getEndTime()); - break; - case DOUBLE: - updateDoubleLastValue( - ((Number) statistics.getLastValue()).doubleValue(), statistics.getEndTime()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - if (statistics instanceof DateStatistics) { - updateBinaryLastValue( - new Binary( - TSDataType.getDateStringValue((Integer) statistics.getLastValue()), - StandardCharsets.UTF_8), - statistics.getEndTime()); - } else { - if (statistics.getLastValue() instanceof Binary) { - updateBinaryLastValue((Binary) statistics.getLastValue(), statistics.getEndTime()); - } else { - updateBinaryLastValue( - new Binary(String.valueOf(statistics.getLastValue()), StandardCharsets.UTF_8), - statistics.getEndTime()); - } - } - break; - case BOOLEAN: - updateBooleanLastValue((boolean) statistics.getLastValue(), statistics.getEndTime()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_LASTVALUE_S_02ECF8E4, - seriesDataType)); - } + strategy.addStatistics(this, statistics, false); } // finalResult should be single column, like: | finalLastValue | @@ -185,37 +93,7 @@ public void setFinal(Column finalResult) { reset(); if (!finalResult.isNull(0)) { initResult = true; - switch (seriesDataType) { - case INT32: - case DATE: - lastValue.setInt(finalResult.getInt(0)); - break; - case INT64: - case TIMESTAMP: - lastValue.setLong(finalResult.getLong(0)); - break; - case FLOAT: - lastValue.setFloat(finalResult.getFloat(0)); - break; - case DOUBLE: - lastValue.setDouble(finalResult.getDouble(0)); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - lastValue.setBinary(finalResult.getBinary(0)); - break; - case BOOLEAN: - lastValue.setBoolean(finalResult.getBoolean(0)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_LASTVALUE_S_02ECF8E4, - seriesDataType)); - } + strategy.setFinal(this, finalResult); } } @@ -230,36 +108,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { columnBuilders[1].appendNull(); return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilders[0].writeInt(lastValue.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilders[0].writeLong(lastValue.getLong()); - break; - case FLOAT: - columnBuilders[0].writeFloat(lastValue.getFloat()); - break; - case DOUBLE: - columnBuilders[0].writeDouble(lastValue.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - columnBuilders[0].writeBinary(lastValue.getBinary()); - break; - case BOOLEAN: - columnBuilders[0].writeBoolean(lastValue.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EXTREME_S_84B651D3, - seriesDataType)); - } + strategy.writeResult(columnBuilders[0], lastValue); columnBuilders[1].writeLong(maxTime); } @@ -269,36 +118,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(lastValue.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(lastValue.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(lastValue.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(lastValue.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - columnBuilder.writeBinary(lastValue.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(lastValue.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EXTREME_S_84B651D3, - seriesDataType)); - } + strategy.writeResult(columnBuilder, lastValue); } @Override @@ -328,19 +148,13 @@ public int getPartialResultSize() { return 2; } - protected void addIntInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateIntLastValue(column[1].getInt(i), column[0].getLong(i)); - } - } + @Override + public TsPrimitiveType getTimeValueResult() { + return lastValue; } - protected void updateIntLastValue(int value, long curTime) { + @Override + public void updateIntResult(final int value, final long curTime) { initResult = true; if (curTime > maxTime) { maxTime = curTime; @@ -348,19 +162,8 @@ protected void updateIntLastValue(int value, long curTime) { } } - protected void addLongInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateLongLastValue(column[1].getLong(i), column[0].getLong(i)); - } - } - } - - protected void updateLongLastValue(long value, long curTime) { + @Override + public void updateLongResult(final long value, final long curTime) { initResult = true; if (curTime > maxTime) { maxTime = curTime; @@ -368,19 +171,8 @@ protected void updateLongLastValue(long value, long curTime) { } } - protected void addFloatInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateFloatLastValue(column[1].getFloat(i), column[0].getLong(i)); - } - } - } - - protected void updateFloatLastValue(float value, long curTime) { + @Override + public void updateFloatResult(final float value, final long curTime) { initResult = true; if (curTime > maxTime) { maxTime = curTime; @@ -388,19 +180,8 @@ protected void updateFloatLastValue(float value, long curTime) { } } - protected void addDoubleInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateDoubleLastValue(column[1].getDouble(i), column[0].getLong(i)); - } - } - } - - protected void updateDoubleLastValue(double value, long curTime) { + @Override + public void updateDoubleResult(final double value, final long curTime) { initResult = true; if (curTime > maxTime) { maxTime = curTime; @@ -408,19 +189,8 @@ protected void updateDoubleLastValue(double value, long curTime) { } } - protected void addBooleanInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateBooleanLastValue(column[1].getBoolean(i), column[0].getLong(i)); - } - } - } - - protected void updateBooleanLastValue(boolean value, long curTime) { + @Override + public void updateBooleanResult(final boolean value, final long curTime) { initResult = true; if (curTime > maxTime) { maxTime = curTime; @@ -428,23 +198,20 @@ protected void updateBooleanLastValue(boolean value, long curTime) { } } - protected void addBinaryInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateBinaryLastValue(column[1].getBinary(i), column[0].getLong(i)); - } - } - } - - protected void updateBinaryLastValue(Binary value, long curTime) { + @Override + public void updateBinaryResult(final Binary value, final long curTime) { initResult = true; if (curTime > maxTime) { maxTime = curTime; lastValue.setBinary(value); } } + + private void ensureSupported() { + if (!strategy.isSupported()) { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, seriesDataType)); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueDescAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueDescAccumulator.java index f83aebf764fb..961b4a957c56 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueDescAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueDescAccumulator.java @@ -35,92 +35,7 @@ public boolean hasFinalResult() { } @Override - protected void addIntInput(Column[] column, BitMap needSkip) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - // skip null value in control column - if (needSkip != null && needSkip.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateIntLastValue(column[1].getInt(i), column[0].getLong(i)); - return; - } - } - } - - @Override - protected void addLongInput(Column[] column, BitMap needSkip) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - // skip null value in control column - if (needSkip != null && needSkip.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateLongLastValue(column[1].getLong(i), column[0].getLong(i)); - return; - } - } - } - - @Override - protected void addFloatInput(Column[] column, BitMap needSkip) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - // skip null value in control column - if (needSkip != null && needSkip.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateFloatLastValue(column[1].getFloat(i), column[0].getLong(i)); - return; - } - } - } - - @Override - protected void addDoubleInput(Column[] column, BitMap needSkip) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - // skip null value in control column - if (needSkip != null && needSkip.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateDoubleLastValue(column[1].getDouble(i), column[0].getLong(i)); - return; - } - } - } - - @Override - protected void addBooleanInput(Column[] column, BitMap needSkip) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - // skip null value in control column - if (needSkip != null && needSkip.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateBooleanLastValue(column[1].getBoolean(i), column[0].getLong(i)); - return; - } - } - } - - @Override - protected void addBinaryInput(Column[] column, BitMap needSkip) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - // skip null value in control column - if (needSkip != null && needSkip.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateBinaryLastValue(column[1].getBinary(i), column[0].getLong(i)); - return; - } - } + public void addInput(final Column[] columns, final BitMap needSkip) { + addInputWithStrategy(columns, needSkip, true, false); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java index fad106ccf285..24cc51c7205d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java @@ -21,12 +21,15 @@ import org.apache.iotdb.calc.execution.aggregation.Accumulator; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.utils.TypeServices; +import org.apache.iotdb.db.utils.TypeServices.Aggregation.MaxMinByAccumulatorStrategy; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; @@ -41,7 +44,8 @@ import static com.google.common.base.Preconditions.checkArgument; /** max(x,y) returns the value of x associated with the maximum value of y over all input values. */ -public abstract class MaxMinByBaseAccumulator implements Accumulator { +public abstract class MaxMinByBaseAccumulator + implements Accumulator, TypeServices.Aggregation.MaxMinByAccumulator { private final TSDataType xDataType; @@ -51,53 +55,33 @@ public abstract class MaxMinByBaseAccumulator implements Accumulator { private final TsPrimitiveType xResult; + private final MaxMinByAccumulatorStrategy xStrategy; + + private final MaxMinByAccumulatorStrategy yStrategy; + private boolean xNull = true; private boolean initResult; private long yTimeStamp = Long.MAX_VALUE; - private static final String UNSUPPORTED_TYPE_MESSAGE = "Unsupported data type in MaxBy/MinBy: %s"; - protected MaxMinByBaseAccumulator(TSDataType xDataType, TSDataType yDataType) { this.xDataType = xDataType; this.yDataType = yDataType; - this.xResult = TsPrimitiveType.getByType(xDataType); - this.yExtremeValue = TsPrimitiveType.getByType(yDataType); + final Type xType = Type.fromTsDataType(xDataType); + final Type yType = Type.fromTsDataType(yDataType); + this.xResult = xType.getTsPrimitiveType(); + this.yExtremeValue = yType.getTsPrimitiveType(); + this.xStrategy = TypeServices.Aggregation.MAX_MIN_BY_ACCUMULATOR_STRATEGY_SERVICE.call(xType); + this.yStrategy = TypeServices.Aggregation.MAX_MIN_BY_ACCUMULATOR_STRATEGY_SERVICE.call(yType); + ensureSupported(); } // Column should be like: | Time | x | y | @Override public void addInput(Column[] column, BitMap bitMap) { - checkArgument( - column.length == 3, - DataNodeQueryMessages - .EXCEPTION_LENGTH_OF_INPUT_COLUMN_LEFT_BRACKET_RIGHT_BRACKET_FOR_MAXBY_SLASH_MINBY_SHOULD_B_1F3F2F1C); - switch (yDataType) { - case INT32: - case DATE: - addIntInput(column, bitMap); - return; - case INT64: - case TIMESTAMP: - addLongInput(column, bitMap); - return; - case FLOAT: - addFloatInput(column, bitMap); - return; - case DOUBLE: - addDoubleInput(column, bitMap); - return; - case STRING: - addBinaryInput(column, bitMap); - return; - case TEXT: - case BLOB: - case BOOLEAN: - case OBJECT: - default: - throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, yDataType)); - } + checkArgument(column.length == 3, "Length of input Column[] for MaxBy/MinBy should be 3"); + yStrategy.addInput(this, column, bitMap); } // partialResult should be like: | partialMaxByBinary | @@ -175,19 +159,8 @@ public TSDataType getFinalType() { return xDataType; } - private void addIntInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[2].isNull(i)) { - updateIntResult(column[0].getLong(i), column[2].getInt(i), column[1], i); - } - } - } - - private void updateIntResult(long time, int yValue, Column xColumn, int xIndex) { + @Override + public void updateIntResult(long time, int yValue, Column xColumn, int xIndex) { if (!initResult || check(yValue, yExtremeValue.getInt()) || (yValue == yExtremeValue.getInt() && time < yTimeStamp)) { @@ -198,19 +171,8 @@ private void updateIntResult(long time, int yValue, Column xColumn, int xIndex) } } - private void addLongInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[2].isNull(i)) { - updateLongResult(column[0].getLong(i), column[2].getLong(i), column[1], i); - } - } - } - - private void updateLongResult(long time, long yValue, Column xColumn, int xIndex) { + @Override + public void updateLongResult(long time, long yValue, Column xColumn, int xIndex) { if (!initResult || check(yValue, yExtremeValue.getLong()) || (yValue == yExtremeValue.getLong() && time < yTimeStamp)) { @@ -221,19 +183,8 @@ private void updateLongResult(long time, long yValue, Column xColumn, int xIndex } } - private void addFloatInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[2].isNull(i)) { - updateFloatResult(column[0].getLong(i), column[2].getFloat(i), column[1], i); - } - } - } - - private void updateFloatResult(long time, float yValue, Column xColumn, int xIndex) { + @Override + public void updateFloatResult(long time, float yValue, Column xColumn, int xIndex) { if (!initResult || check(yValue, yExtremeValue.getFloat()) || (yValue == yExtremeValue.getFloat() && time < yTimeStamp)) { @@ -244,19 +195,8 @@ private void updateFloatResult(long time, float yValue, Column xColumn, int xInd } } - private void addDoubleInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[2].isNull(i)) { - updateDoubleResult(column[0].getLong(i), column[2].getDouble(i), column[1], i); - } - } - } - - private void updateDoubleResult(long time, double yValue, Column xColumn, int xIndex) { + @Override + public void updateDoubleResult(long time, double yValue, Column xColumn, int xIndex) { if (!initResult || check(yValue, yExtremeValue.getDouble()) || (yValue == yExtremeValue.getDouble() && time < yTimeStamp)) { @@ -267,19 +207,8 @@ private void updateDoubleResult(long time, double yValue, Column xColumn, int xI } } - private void addBinaryInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[2].isNull(i)) { - updateBinaryResult(column[0].getLong(i), column[2].getBinary(i), column[1], i); - } - } - } - - private void updateBinaryResult(long time, Binary yValue, Column xColumn, int xIndex) { + @Override + public void updateBinaryResult(long time, Binary yValue, Column xColumn, int xIndex) { if (!initResult || check(yValue, yExtremeValue.getBinary()) || (yValue.compareTo(yExtremeValue.getBinary()) == 0 && time < yTimeStamp)) { @@ -295,33 +224,7 @@ private void writeX(ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (xDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(xResult.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(xResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(xResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(xResult.getDouble()); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - columnBuilder.writeBinary(xResult.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(xResult.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, xDataType)); - } + xStrategy.writeXResult(columnBuilder, xResult); } private void updateX(Column xColumn, int xIndex) { @@ -329,34 +232,7 @@ private void updateX(Column xColumn, int xIndex) { xNull = true; } else { xNull = false; - switch (xDataType) { - case INT32: - case DATE: - xResult.setInt(xColumn.getInt(xIndex)); - break; - case INT64: - case TIMESTAMP: - xResult.setLong(xColumn.getLong(xIndex)); - break; - case FLOAT: - xResult.setFloat(xColumn.getFloat(xIndex)); - break; - case DOUBLE: - xResult.setDouble(xColumn.getDouble(xIndex)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - xResult.setBinary(xColumn.getBinary(xIndex)); - break; - case BOOLEAN: - xResult.setBoolean(xColumn.getBoolean(xIndex)); - break; - default: - throw new UnSupportedDataTypeException( - String.format(UNSUPPORTED_TYPE_MESSAGE, xDataType)); - } + xStrategy.setXResult(xResult, xColumn, xIndex); } } @@ -382,124 +258,35 @@ private byte[] serialize() { private void writeIntermediateToStream( TSDataType dataType, TsPrimitiveType value, DataOutputStream dataOutputStream) throws IOException { - switch (dataType) { - case INT32: - case DATE: - dataOutputStream.writeInt(value.getInt()); - break; - case INT64: - case TIMESTAMP: - dataOutputStream.writeLong(value.getLong()); - break; - case FLOAT: - dataOutputStream.writeFloat(value.getFloat()); - break; - case DOUBLE: - dataOutputStream.writeDouble(value.getDouble()); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - String content = value.getBinary().toString(); - dataOutputStream.writeInt(content.length()); - dataOutputStream.writeBytes(content); - break; - case BOOLEAN: - dataOutputStream.writeBoolean(value.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, dataType)); - } + Type.fromTsDataType(dataType).serialize(value, dataOutputStream); } private void updateFromBytesIntermediateInput(byte[] bytes) { - long time = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); - int offset = Long.BYTES; - // Use Column to store x value - TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(xDataType)); - ColumnBuilder columnBuilder = builder.getValueColumnBuilders()[0]; - switch (yDataType) { - case INT32: - case DATE: - int intMaxVal = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateIntResult(time, intMaxVal, columnBuilder.build(), 0); - break; - case INT64: - case TIMESTAMP: - long longMaxVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - offset += Long.BYTES; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateLongResult(time, longMaxVal, columnBuilder.build(), 0); - break; - case FLOAT: - float floatMaxVal = BytesUtils.bytesToFloat(bytes, offset); - offset += Float.BYTES; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateFloatResult(time, floatMaxVal, columnBuilder.build(), 0); - break; - case DOUBLE: - double doubleMaxVal = BytesUtils.bytesToDouble(bytes, offset); - offset += Long.BYTES; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateDoubleResult(time, doubleMaxVal, columnBuilder.build(), 0); - break; - case STRING: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - Binary binaryMaxVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - offset += length; - readXFromBytesIntermediateInput(bytes, offset, columnBuilder); - updateBinaryResult(time, binaryMaxVal, columnBuilder.build(), 0); - break; - case TEXT: - case BLOB: - case BOOLEAN: - case OBJECT: - default: - throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, yDataType)); - } + final long time = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); + yStrategy.updateIntermediate(this, time, bytes, Long.BYTES); } - private void readXFromBytesIntermediateInput( - byte[] bytes, int offset, ColumnBuilder columnBuilder) { - boolean isXNull = BytesUtils.bytesToBool(bytes, offset); - offset += 1; + @Override + public Column readXFromBytesIntermediateInput(byte[] bytes, int offset) { + // Use Column to preserve the existing null handling when updating the selected x value. + final TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(xDataType)); + final ColumnBuilder columnBuilder = builder.getValueColumnBuilders()[0]; + final boolean isXNull = BytesUtils.bytesToBool(bytes, offset); if (isXNull) { columnBuilder.appendNull(); } else { - switch (xDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(BytesUtils.bytesToInt(bytes, offset)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(BytesUtils.bytesToLongFromOffset(bytes, 8, offset)); - break; - case FLOAT: - columnBuilder.writeFloat(BytesUtils.bytesToFloat(bytes, offset)); - break; - case DOUBLE: - columnBuilder.writeDouble(BytesUtils.bytesToDouble(bytes, offset)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - columnBuilder.writeBinary(new Binary(BytesUtils.subBytes(bytes, offset, length))); - break; - case BOOLEAN: - columnBuilder.writeBoolean(BytesUtils.bytesToBool(bytes, offset)); - break; - default: - throw new UnSupportedDataTypeException( - String.format(UNSUPPORTED_TYPE_MESSAGE, xDataType)); - } + xStrategy.writeSerializedValue(bytes, offset + Byte.BYTES, columnBuilder); + } + return columnBuilder.build(); + } + + private void ensureSupported() { + if (!xStrategy.isXSupported() || !yStrategy.isYSupported()) { + final TSDataType unsupportedType = xStrategy.isXSupported() ? yDataType : xDataType; + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MAXBY_MINBY_ARG_DD30FAB1, + unsupportedType)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java index 825c51c80243..50beb20f9898 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java @@ -21,11 +21,15 @@ import org.apache.iotdb.calc.execution.aggregation.Accumulator; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.utils.TypeServices; +import org.apache.iotdb.db.utils.TypeServices.Aggregation.ExtremeValueAccumulator; +import org.apache.iotdb.db.utils.TypeServices.Aggregation.ExtremeValueAccumulatorStrategy; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.TsPrimitiveType; @@ -33,48 +37,25 @@ import static com.google.common.base.Preconditions.checkArgument; -public class MaxValueAccumulator implements Accumulator { +public class MaxValueAccumulator implements Accumulator, ExtremeValueAccumulator { private final TSDataType seriesDataType; private final TsPrimitiveType maxResult; + private final ExtremeValueAccumulatorStrategy strategy; private boolean initResult; public MaxValueAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - this.maxResult = TsPrimitiveType.getByType(seriesDataType); + final Type type = Type.fromTsDataType(seriesDataType); + this.maxResult = type.getTsPrimitiveType(); + this.strategy = TypeServices.Aggregation.EXTREME_VALUE_ACCUMULATOR_STRATEGY_SERVICE.call(type); + ensureSupported(); } // Column should be like: | Time | Value | @Override public void addInput(Column[] columns, BitMap bitMap) { - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(columns, bitMap); - return; - case INT64: - case TIMESTAMP: - addLongInput(columns, bitMap); - return; - case FLOAT: - addFloatInput(columns, bitMap); - return; - case DOUBLE: - addDoubleInput(columns, bitMap); - return; - case STRING: - addBinaryInput(columns, bitMap); - return; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MAXVALUE_S_521AC345, - seriesDataType)); - } + strategy.addInput(this, columns, bitMap); } // partialResult should be like: | partialMaxValue1 | @@ -86,34 +67,7 @@ public void addIntermediate(Column[] partialResult) { if (partialResult[0].isNull(0)) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntResult(partialResult[0].getInt(0)); - break; - case INT64: - case TIMESTAMP: - updateLongResult(partialResult[0].getLong(0)); - break; - case FLOAT: - updateFloatResult(partialResult[0].getFloat(0)); - break; - case DOUBLE: - updateDoubleResult(partialResult[0].getDouble(0)); - break; - case STRING: - updateBinaryResult(partialResult[0].getBinary(0)); - break; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MAXVALUE_S_521AC345, - seriesDataType)); - } + strategy.addIntermediate(this, partialResult[0]); } @Override @@ -121,34 +75,7 @@ public void addStatistics(Statistics statistics) { if (statistics == null) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntResult(((Number) statistics.getMaxValue()).intValue()); - break; - case INT64: - case TIMESTAMP: - updateLongResult(((Number) statistics.getMaxValue()).longValue()); - break; - case FLOAT: - updateFloatResult(((Number) statistics.getMaxValue()).floatValue()); - break; - case DOUBLE: - updateDoubleResult(((Number) statistics.getMaxValue()).doubleValue()); - break; - case STRING: - updateBinaryResult((Binary) statistics.getMaxValue()); - break; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MAXVALUE_S_521AC345, - seriesDataType)); - } + strategy.addStatistics(this, statistics.getMaxValue()); } // finalResult should be single column, like: | finalCountValue | @@ -158,34 +85,7 @@ public void setFinal(Column finalResult) { return; } initResult = true; - switch (seriesDataType) { - case INT32: - case DATE: - maxResult.setInt(finalResult.getInt(0)); - break; - case INT64: - case TIMESTAMP: - maxResult.setLong(finalResult.getLong(0)); - break; - case FLOAT: - maxResult.setFloat(finalResult.getFloat(0)); - break; - case DOUBLE: - maxResult.setDouble(finalResult.getDouble(0)); - break; - case STRING: - maxResult.setBinary(finalResult.getBinary(0)); - break; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MAXVALUE_S_521AC345, - seriesDataType)); - } + strategy.setFinal(this, finalResult); } // columnBuilder should be single in countAccumulator @@ -198,34 +98,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { columnBuilders[0].appendNull(); return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilders[0].writeInt(maxResult.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilders[0].writeLong(maxResult.getLong()); - break; - case FLOAT: - columnBuilders[0].writeFloat(maxResult.getFloat()); - break; - case DOUBLE: - columnBuilders[0].writeDouble(maxResult.getDouble()); - break; - case STRING: - columnBuilders[0].writeBinary(maxResult.getBinary()); - break; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MAXVALUE_S_521AC345, - seriesDataType)); - } + strategy.writeResult(columnBuilders[0], maxResult); } @Override @@ -234,34 +107,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(maxResult.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(maxResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(maxResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(maxResult.getDouble()); - break; - case STRING: - columnBuilder.writeBinary(maxResult.getBinary()); - break; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MAXVALUE_S_521AC345, - seriesDataType)); - } + strategy.writeResult(columnBuilder, maxResult); } @Override @@ -285,98 +131,56 @@ public TSDataType getFinalType() { return maxResult.getDataType(); } - private void addIntInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateIntResult(column[1].getInt(i)); - } - } + @Override + public TsPrimitiveType getResult() { + return maxResult; } - private void updateIntResult(int maxVal) { - if (!initResult || maxVal > maxResult.getInt()) { + @Override + public void updateIntResult(final int value) { + if (!initResult || value > maxResult.getInt()) { initResult = true; - maxResult.setInt(maxVal); - } - } - - private void addLongInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateLongResult(column[1].getLong(i)); - } + maxResult.setInt(value); } } - private void updateLongResult(long maxVal) { - if (!initResult || maxVal > maxResult.getLong()) { + @Override + public void updateLongResult(final long value) { + if (!initResult || value > maxResult.getLong()) { initResult = true; - maxResult.setLong(maxVal); - } - } - - private void addFloatInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateFloatResult(column[1].getFloat(i)); - } + maxResult.setLong(value); } } - private void updateFloatResult(float maxVal) { - if (!initResult || maxVal > maxResult.getFloat()) { + @Override + public void updateFloatResult(final float value) { + if (!initResult || value > maxResult.getFloat()) { initResult = true; - maxResult.setFloat(maxVal); - } - } - - private void addDoubleInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateDoubleResult(column[1].getDouble(i)); - } + maxResult.setFloat(value); } } - private void updateDoubleResult(double maxVal) { - if (!initResult || maxVal > maxResult.getDouble()) { + @Override + public void updateDoubleResult(final double value) { + if (!initResult || value > maxResult.getDouble()) { initResult = true; - maxResult.setDouble(maxVal); + maxResult.setDouble(value); } } - private void addBinaryInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateBinaryResult(column[1].getBinary(i)); - } + @Override + public void updateBinaryResult(final Binary value) { + if (!initResult || value.compareTo(maxResult.getBinary()) > 0) { + initResult = true; + maxResult.setBinary(value); } } - private void updateBinaryResult(Binary maxVal) { - if (!initResult || maxVal.compareTo(maxResult.getBinary()) > 0) { - initResult = true; - maxResult.setBinary(maxVal); + private void ensureSupported() { + if (!strategy.isSupported()) { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, seriesDataType)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java index 9d35b77fb072..f18b92f7ebfc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java @@ -21,11 +21,15 @@ import org.apache.iotdb.calc.execution.aggregation.Accumulator; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.utils.TypeServices; +import org.apache.iotdb.db.utils.TypeServices.Aggregation.ExtremeValueAccumulator; +import org.apache.iotdb.db.utils.TypeServices.Aggregation.ExtremeValueAccumulatorStrategy; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.TsPrimitiveType; @@ -33,48 +37,25 @@ import static com.google.common.base.Preconditions.checkArgument; -public class MinValueAccumulator implements Accumulator { +public class MinValueAccumulator implements Accumulator, ExtremeValueAccumulator { private final TSDataType seriesDataType; private final TsPrimitiveType minResult; + private final ExtremeValueAccumulatorStrategy strategy; private boolean initResult = false; public MinValueAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - this.minResult = TsPrimitiveType.getByType(seriesDataType); + final Type type = Type.fromTsDataType(seriesDataType); + this.minResult = type.getTsPrimitiveType(); + this.strategy = TypeServices.Aggregation.EXTREME_VALUE_ACCUMULATOR_STRATEGY_SERVICE.call(type); + ensureSupported(); } // Column should be like: | Time | Value | @Override public void addInput(Column[] columns, BitMap bitMap) { - switch (seriesDataType) { - case INT32: - case DATE: - addIntInput(columns, bitMap); - return; - case INT64: - case TIMESTAMP: - addLongInput(columns, bitMap); - return; - case FLOAT: - addFloatInput(columns, bitMap); - return; - case DOUBLE: - addDoubleInput(columns, bitMap); - return; - case STRING: - addBinaryInput(columns, bitMap); - return; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MINVALUE_S_BC092694, - seriesDataType)); - } + strategy.addInput(this, columns, bitMap); } // partialResult should be like: | partialMinValue1 | @@ -86,34 +67,7 @@ public void addIntermediate(Column[] partialResult) { if (partialResult[0].isNull(0)) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntResult(partialResult[0].getInt(0)); - break; - case INT64: - case TIMESTAMP: - updateLongResult(partialResult[0].getLong(0)); - break; - case FLOAT: - updateFloatResult(partialResult[0].getFloat(0)); - break; - case DOUBLE: - updateDoubleResult(partialResult[0].getDouble(0)); - break; - case STRING: - updateBinaryResult(partialResult[0].getBinary(0)); - break; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MINVALUE_S_BC092694, - seriesDataType)); - } + strategy.addIntermediate(this, partialResult[0]); } @Override @@ -121,34 +75,7 @@ public void addStatistics(Statistics statistics) { if (statistics == null) { return; } - switch (seriesDataType) { - case INT32: - case DATE: - updateIntResult(((Number) statistics.getMinValue()).intValue()); - break; - case INT64: - case TIMESTAMP: - updateLongResult(((Number) statistics.getMinValue()).longValue()); - break; - case FLOAT: - updateFloatResult(((Number) statistics.getMinValue()).floatValue()); - break; - case DOUBLE: - updateDoubleResult(((Number) statistics.getMinValue()).doubleValue()); - break; - case STRING: - updateBinaryResult((Binary) statistics.getMinValue()); - break; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MINVALUE_S_BC092694, - seriesDataType)); - } + strategy.addStatistics(this, statistics.getMinValue()); } // finalResult should be single column, like: | finalCountValue | @@ -158,34 +85,7 @@ public void setFinal(Column finalResult) { return; } initResult = true; - switch (seriesDataType) { - case INT32: - case DATE: - minResult.setInt(finalResult.getInt(0)); - break; - case INT64: - case TIMESTAMP: - minResult.setLong(finalResult.getLong(0)); - break; - case FLOAT: - minResult.setFloat(finalResult.getFloat(0)); - break; - case DOUBLE: - minResult.setDouble(finalResult.getDouble(0)); - break; - case STRING: - minResult.setBinary(finalResult.getBinary(0)); - break; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MINVALUE_S_BC092694, - seriesDataType)); - } + strategy.setFinal(this, finalResult); } // columnBuilder should be single in MinValueAccumulator @@ -198,34 +98,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { columnBuilders[0].appendNull(); return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilders[0].writeInt(minResult.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilders[0].writeLong(minResult.getLong()); - break; - case FLOAT: - columnBuilders[0].writeFloat(minResult.getFloat()); - break; - case DOUBLE: - columnBuilders[0].writeDouble(minResult.getDouble()); - break; - case STRING: - columnBuilders[0].writeBinary(minResult.getBinary()); - break; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MINVALUE_S_BC092694, - seriesDataType)); - } + strategy.writeResult(columnBuilders[0], minResult); } @Override @@ -234,34 +107,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { columnBuilder.appendNull(); return; } - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(minResult.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(minResult.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(minResult.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(minResult.getDouble()); - break; - case STRING: - columnBuilder.writeBinary(minResult.getBinary()); - break; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_MINVALUE_S_BC092694, - seriesDataType)); - } + strategy.writeResult(columnBuilder, minResult); } @Override @@ -285,99 +131,56 @@ public TSDataType getFinalType() { return minResult.getDataType(); } - private void addIntInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateIntResult(column[1].getInt(i)); - } - } + @Override + public TsPrimitiveType getResult() { + return minResult; } - private void updateIntResult(int minVal) { - if (!initResult || minVal < minResult.getInt()) { + @Override + public void updateIntResult(final int value) { + if (!initResult || value < minResult.getInt()) { initResult = true; - minResult.setInt(minVal); - } - } - - private void addLongInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - // skip null value in control column - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateLongResult(column[1].getLong(i)); - } + minResult.setInt(value); } } - private void updateLongResult(long minVal) { - if (!initResult || minVal < minResult.getLong()) { + @Override + public void updateLongResult(final long value) { + if (!initResult || value < minResult.getLong()) { initResult = true; - minResult.setLong(minVal); - } - } - - private void addFloatInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateFloatResult(column[1].getFloat(i)); - } + minResult.setLong(value); } } - private void updateFloatResult(float minVal) { - if (!initResult || minVal < minResult.getFloat()) { + @Override + public void updateFloatResult(final float value) { + if (!initResult || value < minResult.getFloat()) { initResult = true; - minResult.setFloat(minVal); - } - } - - private void addDoubleInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateDoubleResult(column[1].getDouble(i)); - } + minResult.setFloat(value); } } - private void updateDoubleResult(double minVal) { - if (!initResult || minVal < minResult.getDouble()) { + @Override + public void updateDoubleResult(final double value) { + if (!initResult || value < minResult.getDouble()) { initResult = true; - minResult.setDouble(minVal); + minResult.setDouble(value); } } - private void addBinaryInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - updateBinaryResult(column[1].getBinary(i)); - } + @Override + public void updateBinaryResult(final Binary value) { + if (!initResult || value.compareTo(minResult.getBinary()) < 0) { + initResult = true; + minResult.setBinary(value); } } - private void updateBinaryResult(Binary minVal) { - if (!initResult || minVal.compareTo(minResult.getBinary()) < 0) { - initResult = true; - minResult.setBinary(minVal); + private void ensureSupported() { + if (!strategy.isSupported()) { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, seriesDataType)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java index 66928b24f959..0224772f90cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java @@ -45,32 +45,28 @@ public SumAccumulator(TSDataType seriesDataType) { // Column should be like: | Time | Value | @Override public void addInput(Column[] columns, BitMap bitMap) { - switch (seriesDataType) { - case INT32: - addIntInput(columns, bitMap); - return; - case INT64: - addLongInput(columns, bitMap); - return; - case FLOAT: - addFloatInput(columns, bitMap); - return; - case DOUBLE: - addDoubleInput(columns, bitMap); - return; - case TEXT: - case BLOB: - case OBJECT: - case BOOLEAN: - case TIMESTAMP: - case DATE: - case STRING: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_AGGREGATION_AVG_S_D1DAD6A6, - seriesDataType)); + final boolean isNumeric; + try { + isNumeric = seriesDataType.isNumeric(); + } catch (final UnSupportedDataTypeException ignored) { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, seriesDataType)); + } + if (!isNumeric) { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, seriesDataType)); + } + final int count = columns[0].getPositionCount(); + for (int i = 0; i < count; i++) { + if (bitMap != null && !bitMap.isMarked(i)) { + continue; + } + if (!columns[1].isNull(i)) { + initResult = true; + sumValue += columns[1].getDouble(i); + } } } @@ -163,56 +159,4 @@ public TSDataType[] getIntermediateType() { public TSDataType getFinalType() { return TSDataType.DOUBLE; } - - private void addIntInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - initResult = true; - sumValue += column[1].getInt(i); - } - } - } - - private void addLongInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - initResult = true; - sumValue += column[1].getLong(i); - } - } - } - - private void addFloatInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - initResult = true; - sumValue += column[1].getFloat(i); - } - } - } - - private void addDoubleInput(Column[] column, BitMap bitMap) { - int count = column[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (bitMap != null && !bitMap.isMarked(i)) { - continue; - } - if (!column[1].isNull(i)) { - initResult = true; - sumValue += column[1].getDouble(i); - } - } - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java index d1e1c6a28876..013ef4f920e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.queryengine.execution.operator; import org.apache.iotdb.commons.queryengine.plan.udf.BuiltinAggregationFunction; -import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.execution.aggregation.TreeAggregator; import org.apache.iotdb.db.queryengine.execution.aggregation.timerangeiterator.ITimeRangeIterator; import org.apache.iotdb.db.queryengine.execution.aggregation.timerangeiterator.SingleTimeWindowIterator; @@ -30,7 +29,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.AggregationDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.GroupByTimeParameter; -import org.apache.iotdb.db.queryengine.statistics.StatisticsManager; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -39,13 +38,9 @@ import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; -import org.apache.tsfile.read.common.block.column.BooleanColumn; -import org.apache.tsfile.read.common.block.column.DoubleColumn; -import org.apache.tsfile.read.common.block.column.FloatColumn; -import org.apache.tsfile.read.common.block.column.IntColumn; -import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Pair; import java.time.ZoneId; @@ -248,28 +243,9 @@ public static long calculateMaxAggregationResultSizeForLastQuery( } public static long getOutputColumnSizePerLine(TSDataType tsDataType) { - switch (tsDataType) { - case INT32: - case DATE: - return IntColumn.SIZE_IN_BYTES_PER_POSITION; - case INT64: - case TIMESTAMP: - return LongColumn.SIZE_IN_BYTES_PER_POSITION; - case FLOAT: - return FloatColumn.SIZE_IN_BYTES_PER_POSITION; - case DOUBLE: - return DoubleColumn.SIZE_IN_BYTES_PER_POSITION; - case BOOLEAN: - return BooleanColumn.SIZE_IN_BYTES_PER_POSITION; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - return StatisticsManager.getInstance().getMaxBinarySizeInBytes(); - default: - throw new UnsupportedOperationException( - DataNodeQueryMessages.UNKNOWN_DATA_TYPE_2 + tsDataType); - } + return TypeServices.Aggregation.OUTPUT_COLUMN_SIZE_PER_LINE_SERVICE + .call(Type.fromTsDataType(tsDataType)) + .getAsLong(); } public static String addPartialSuffix(String aggregationName) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/InsertTabletStatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/InsertTabletStatementGenerator.java index 3784857e5f29..d32f8bca7d5c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/InsertTabletStatementGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/InsertTabletStatementGenerator.java @@ -21,8 +21,8 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.parameter.InputLocation; -import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; @@ -32,7 +32,6 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.util.Arrays; @@ -73,35 +72,12 @@ public void reset() { this.times = new long[rowLimit]; this.columns = new Object[this.measurements.length]; for (int i = 0; i < this.measurements.length; i++) { - switch (dataTypes[i]) { - case BOOLEAN: - columns[i] = new boolean[rowLimit]; - break; - case INT32: - case DATE: - columns[i] = new int[rowLimit]; - break; - case INT64: - case TIMESTAMP: - columns[i] = new long[rowLimit]; - break; - case FLOAT: - columns[i] = new float[rowLimit]; - break; - case DOUBLE: - columns[i] = new double[rowLimit]; - break; - case TEXT: - case STRING: - case BLOB: - columns[i] = new Binary[rowLimit]; - Arrays.fill((Binary[]) columns[i], Binary.EMPTY_VALUE); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataTypes[i])); + columns[i] = + TypeServices.StorageEngine.PRIMITIVE_ARRAY_ALLOCATOR_SERVICE + .call(Type.fromTsDataType(dataTypes[i])) + .apply(rowLimit); + if (dataTypes[i].isBinary()) { + Arrays.fill((Binary[]) columns[i], Binary.EMPTY_VALUE); } } this.bitMaps = new BitMap[this.measurements.length]; @@ -127,35 +103,12 @@ public InsertTabletStatement constructInsertTabletStatement() { times = Arrays.copyOf(times, rowCount); for (int i = 0; i < columns.length; i++) { bitMaps[i] = bitMaps[i].getRegion(0, rowCount); - switch (dataTypes[i]) { - case BOOLEAN: - columns[i] = Arrays.copyOf((boolean[]) columns[i], rowCount); - break; - case INT32: - case DATE: - columns[i] = Arrays.copyOf((int[]) columns[i], rowCount); - break; - case INT64: - case TIMESTAMP: - columns[i] = Arrays.copyOf((long[]) columns[i], rowCount); - break; - case FLOAT: - columns[i] = Arrays.copyOf((float[]) columns[i], rowCount); - break; - case DOUBLE: - columns[i] = Arrays.copyOf((double[]) columns[i], rowCount); - break; - case TEXT: - case STRING: - case BLOB: - columns[i] = Arrays.copyOf((Binary[]) columns[i], rowCount); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataTypes[i])); - } + Object copiedColumn = + TypeServices.StorageEngine.PRIMITIVE_ARRAY_ALLOCATOR_SERVICE + .call(Type.fromTsDataType(dataTypes[i])) + .apply(rowCount); + System.arraycopy(columns[i], 0, copiedColumn, 0, rowCount); + columns[i] = copiedColumn; } } @@ -199,36 +152,9 @@ protected void processColumn( TSDataType dataType, Type sourceTypeConvertor, int rowIndex) { - switch (dataType) { - case INT32: - case DATE: - ((int[]) columns)[rowCount] = sourceTypeConvertor.getInt(valueColumn, rowIndex); - break; - case INT64: - case TIMESTAMP: - ((long[]) columns)[rowCount] = sourceTypeConvertor.getLong(valueColumn, rowIndex); - break; - case FLOAT: - ((float[]) columns)[rowCount] = sourceTypeConvertor.getFloat(valueColumn, rowIndex); - break; - case DOUBLE: - ((double[]) columns)[rowCount] = sourceTypeConvertor.getDouble(valueColumn, rowIndex); - break; - case BOOLEAN: - ((boolean[]) columns)[rowCount] = sourceTypeConvertor.getBoolean(valueColumn, rowIndex); - break; - case TEXT: - case BLOB: - case STRING: - ((Binary[]) columns)[rowCount] = sourceTypeConvertor.getBinary(valueColumn, rowIndex); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_WHEN_CONVERT_DATA_AT_CLIENT_405429CC, - valueColumn.getDataType())); - } + TypeServices.StorageEngine.SOURCE_COLUMN_TO_TABLET_VALUE_WRITER_SERVICE + .call(Type.fromTsDataType(dataType)) + .write(sourceTypeConvertor, valueColumn, rowIndex, columns, rowCount); } protected long sizeOf(Object[] arr, Class clazz) { @@ -259,36 +185,7 @@ private long getColumnsBytes() { } long bytes = 0L; for (int i = 0; i < columns.length; i++) { - switch (dataTypes[i]) { - case INT32: - case DATE: - bytes += RamUsageEstimator.sizeOf((int[]) columns[i]); - break; - case INT64: - case TIMESTAMP: - bytes += RamUsageEstimator.sizeOf((long[]) columns[i]); - break; - case FLOAT: - bytes += RamUsageEstimator.sizeOf((float[]) columns[i]); - break; - case DOUBLE: - bytes += RamUsageEstimator.sizeOf((double[]) columns[i]); - break; - case BOOLEAN: - bytes += RamUsageEstimator.sizeOf((boolean[]) columns[i]); - break; - case TEXT: - case BLOB: - case STRING: - bytes += RamUsageEstimator.sizeOf((Binary[]) columns[i]); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_WHEN_CONVERT_DATA_AT_CLIENT_405429CC, - dataTypes[i])); - } + bytes += Type.fromTsDataType(dataTypes[i]).estimateArraySize(columns[i]); } return bytes; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java index 65a992891205..e2c7984b4999 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java @@ -38,6 +38,7 @@ import org.apache.iotdb.db.queryengine.transformation.dag.input.QueryDataSetInputLayer; import org.apache.iotdb.db.queryengine.transformation.dag.input.TsBlockInputDataSet; import org.apache.iotdb.db.queryengine.transformation.dag.udf.UDTFContext; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.db.utils.datastructure.TimeSelector; import com.google.common.util.concurrent.ListenableFuture; @@ -48,6 +49,7 @@ import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -329,37 +331,18 @@ protected YieldableState collectDataPoint(ColumnBuilder writer, long currentTime if (valueColumn.isNull(currentIndex)) { writer.appendNull(); } else { - TSDataType type = transformers[index].getDataTypes()[0]; - switch (type) { - case INT32: - case DATE: - writer.writeInt(valueColumn.getInt(currentIndex)); - break; - case INT64: - case TIMESTAMP: - writer.writeLong(valueColumn.getLong(currentIndex)); - break; - case FLOAT: - writer.writeFloat(valueColumn.getFloat(currentIndex)); - break; - case DOUBLE: - writer.writeDouble(valueColumn.getDouble(currentIndex)); - break; - case BOOLEAN: - writer.writeBoolean(valueColumn.getBoolean(currentIndex)); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - writer.writeBinary(valueColumn.getBinary(currentIndex)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - type)); + final TSDataType dataType = transformers[index].getDataTypes()[0]; + final Type type; + try { + type = Type.fromTsDataType(dataType); + } catch (final UnsupportedOperationException ignored) { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, dataType)); } + TypeServices.Transformation.TRANSFORM_COLUMN_VALUE_WRITER_SERVICE + .call(type) + .write(writer, valueColumn, currentIndex); } shouldIterateReadersToNextValid[index] = true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java index 1b8ca2b1020c..3dd64db4c2ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java @@ -45,6 +45,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.SchemaUtils; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.db.utils.datastructure.MemPointIterator; import org.apache.tsfile.block.column.Column; @@ -61,12 +62,7 @@ import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.TsBlockUtil; -import org.apache.tsfile.read.common.block.column.BinaryColumn; -import org.apache.tsfile.read.common.block.column.BooleanColumn; -import org.apache.tsfile.read.common.block.column.DoubleColumn; -import org.apache.tsfile.read.common.block.column.FloatColumn; -import org.apache.tsfile.read.common.block.column.IntColumn; -import org.apache.tsfile.read.common.block.column.LongColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.controller.IChunkLoader; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.IPageReader; @@ -75,17 +71,14 @@ import org.apache.tsfile.read.reader.page.TablePageReader; import org.apache.tsfile.read.reader.series.PaginationController; import org.apache.tsfile.utils.Accountable; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.io.Serializable; -import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Comparator; import java.util.LinkedList; @@ -111,6 +104,7 @@ public class SeriesScanUtil implements Accountable { private final IDeviceID deviceID; protected boolean isAligned = false; private final TSDataType dataType; + private final Type typeInterface; // inner class of SeriesReader for order purpose protected final TimeOrderUtils orderUtils; @@ -169,6 +163,7 @@ public SeriesScanUtil( this.seriesPath = seriesPath; this.deviceID = seriesPath.getDeviceId(); this.dataType = seriesPath.getSeriesType(); + this.typeInterface = Type.fromTsDataType(dataType); this.scanOptions = scanOptions; this.paginationController = scanOptions.getPaginationController(); @@ -1035,439 +1030,11 @@ private TsBlock getTransferedDataTypeTsBlock(TsBlock tsBlock) { int positionCount = tsBlock.getPositionCount(); Column[] newValueColumns = new Column[length]; for (int i = 0; i < length; i++) { - TSDataType sourceType = valueColumns[i].getDataType(); TSDataType finalDataType = getTsDataTypeList().get(i); - switch (finalDataType) { - case BOOLEAN: - if (sourceType == TSDataType.BOOLEAN) { - newValueColumns[i] = valueColumns[i]; - } else { - newValueColumns[i] = - new BooleanColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new boolean[positionCount]); - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = true; - } - } - break; - case INT32: - if (sourceType == TSDataType.INT32) { - newValueColumns[i] = valueColumns[i]; - } else { - newValueColumns[i] = - new IntColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new int[positionCount], - TSDataType.INT32); - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = true; - } - } - break; - case INT64: - if (sourceType == TSDataType.INT64) { - newValueColumns[i] = valueColumns[i]; - } else if (sourceType == TSDataType.INT32) { - newValueColumns[i] = - new LongColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new long[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getLongs()[j] = - ((Number) valueColumns[i].getInts()[j]).longValue(); - } - } - } else if (sourceType == TSDataType.TIMESTAMP) { - newValueColumns[i] = valueColumns[i]; - } else { - newValueColumns[i] = - new LongColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new long[positionCount]); - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = true; - } - } - break; - case FLOAT: - if (sourceType == TSDataType.FLOAT) { - newValueColumns[i] = valueColumns[i]; - } else if (sourceType == TSDataType.INT32) { - newValueColumns[i] = - new FloatColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new float[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getFloats()[j] = - ((Number) valueColumns[i].getInts()[j]).floatValue(); - } - } - } else { - newValueColumns[i] = - new FloatColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new float[positionCount]); - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = true; - } - } - break; - case DOUBLE: - if (sourceType == TSDataType.DOUBLE) { - newValueColumns[i] = valueColumns[i]; - } else if (sourceType == TSDataType.INT32) { - newValueColumns[i] = - new DoubleColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new double[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getDoubles()[j] = - ((Number) valueColumns[i].getInts()[j]).doubleValue(); - } - } - } else if (sourceType == TSDataType.INT64) { - newValueColumns[i] = - new DoubleColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new double[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getDoubles()[j] = - ((Number) valueColumns[i].getLongs()[j]).doubleValue(); - } - } - } else if (sourceType == TSDataType.FLOAT) { - newValueColumns[i] = - new DoubleColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new double[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getDoubles()[j] = - ((Number) valueColumns[i].getFloats()[j]).doubleValue(); - } - } - } else if (sourceType == TSDataType.TIMESTAMP) { - newValueColumns[i] = - new DoubleColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new double[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getDoubles()[j] = - ((Number) valueColumns[i].getLongs()[j]).doubleValue(); - } - } - } else { - newValueColumns[i] = - new DoubleColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new double[positionCount]); - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = true; - } - } - break; - case TEXT: - if (SchemaUtils.isUsingSameColumn(sourceType, TSDataType.TEXT)) { - newValueColumns[i] = valueColumns[i]; - } else if (sourceType == TSDataType.INT32) { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getBinaries()[j] = - new Binary( - String.valueOf(valueColumns[i].getInts()[j]), StandardCharsets.UTF_8); - } - } - } else if (sourceType == TSDataType.DATE) { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getBinaries()[j] = - new Binary( - TSDataType.getDateStringValue(valueColumns[i].getInts()[j]), - StandardCharsets.UTF_8); - } - } - } else if (sourceType == TSDataType.INT64 || sourceType == TSDataType.TIMESTAMP) { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getBinaries()[j] = - new Binary( - String.valueOf(valueColumns[i].getLongs()[j]), StandardCharsets.UTF_8); - } - } - } else if (sourceType == TSDataType.FLOAT) { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getBinaries()[j] = - new Binary( - String.valueOf(valueColumns[i].getFloats()[j]), StandardCharsets.UTF_8); - } - } - } else if (sourceType == TSDataType.DOUBLE) { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getBinaries()[j] = - new Binary( - String.valueOf(valueColumns[i].getDoubles()[j]), StandardCharsets.UTF_8); - } - } - } else if (sourceType == TSDataType.BOOLEAN) { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getBinaries()[j] = - new Binary( - String.valueOf(valueColumns[i].getBooleans()[j]), StandardCharsets.UTF_8); - } - } - } else { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = true; - } - } - break; - case TIMESTAMP: - if (SchemaUtils.isUsingSameColumn(sourceType, TSDataType.TIMESTAMP)) { - newValueColumns[i] = valueColumns[i]; - } else if (sourceType == TSDataType.INT32) { - newValueColumns[i] = - new LongColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new long[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getLongs()[j] = - ((Number) valueColumns[i].getInts()[j]).longValue(); - } - } - } else if (sourceType == TSDataType.INT64) { - newValueColumns[i] = valueColumns[i]; - } else { - newValueColumns[i] = - new LongColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new long[positionCount]); - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = true; - } - } - break; - case DATE: - if (SchemaUtils.isUsingSameColumn(sourceType, TSDataType.DATE)) { - newValueColumns[i] = valueColumns[i]; - } else { - newValueColumns[i] = - new IntColumn( - positionCount, Optional.of(new boolean[positionCount]), new int[positionCount]); - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = true; - } - } - break; - case BLOB: - if (SchemaUtils.isUsingSameColumn(sourceType, TSDataType.BLOB)) { - newValueColumns[i] = valueColumns[i]; - } else { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = true; - } - } - break; - case STRING: - if (SchemaUtils.isUsingSameColumn(sourceType, TSDataType.STRING)) { - newValueColumns[i] = valueColumns[i]; - } else if (sourceType == TSDataType.INT32) { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getBinaries()[j] = - new Binary( - String.valueOf(valueColumns[i].getInts()[j]), StandardCharsets.UTF_8); - } - } - } else if (sourceType == TSDataType.DATE) { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getBinaries()[j] = - new Binary( - TSDataType.getDateStringValue(valueColumns[i].getInts()[j]), - StandardCharsets.UTF_8); - } - } - } else if (sourceType == TSDataType.INT64 || sourceType == TSDataType.TIMESTAMP) { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getBinaries()[j] = - new Binary( - String.valueOf(valueColumns[i].getLongs()[j]), StandardCharsets.UTF_8); - } - } - } else if (sourceType == TSDataType.FLOAT) { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getBinaries()[j] = - new Binary( - String.valueOf(valueColumns[i].getFloats()[j]), StandardCharsets.UTF_8); - } - } - } else if (sourceType == TSDataType.DOUBLE) { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getBinaries()[j] = - new Binary( - String.valueOf(valueColumns[i].getDoubles()[j]), StandardCharsets.UTF_8); - } - } - } else if (sourceType == TSDataType.BOOLEAN) { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; - if (!valueColumns[i].isNull()[j]) { - newValueColumns[i].getBinaries()[j] = - new Binary( - String.valueOf(valueColumns[i].getBooleans()[j]), StandardCharsets.UTF_8); - } - } - } else { - newValueColumns[i] = - new BinaryColumn( - positionCount, - Optional.of(new boolean[positionCount]), - new Binary[positionCount]); - for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { - newValueColumns[i].isNull()[j] = true; - } - } - break; - case OBJECT: - newValueColumns[i] = valueColumns[i]; - case VECTOR: - case UNKNOWN: - default: - break; - } + newValueColumns[i] = + TypeServices.Transformation.ALTERED_DATA_TYPE_COLUMN_TRANSFORMER_SERVICE + .call(Type.fromTsDataType(finalDataType)) + .transform(valueColumns[i], positionCount); } tsBlock = new TsBlock(tsBlock.getTimeColumn(), newValueColumns); @@ -1729,51 +1296,18 @@ private long calculateInitialEndPointTime(final long currentReadStopTime) { private void addTimeValuePairToResult(TimeValuePair timeValuePair, TsBlockBuilder builder) { builder.getTimeColumnBuilder().writeLong(timeValuePair.getTimestamp()); - switch (dataType) { - case BOOLEAN: - builder.getColumnBuilder(0).writeBoolean(timeValuePair.getValue().getBoolean()); - break; - case INT32: - case DATE: - builder.getColumnBuilder(0).writeInt(timeValuePair.getValue().getInt()); - break; - case INT64: - case TIMESTAMP: - builder.getColumnBuilder(0).writeLong(timeValuePair.getValue().getLong()); - break; - case FLOAT: - builder.getColumnBuilder(0).writeFloat(timeValuePair.getValue().getFloat()); - break; - case DOUBLE: - builder.getColumnBuilder(0).writeDouble(timeValuePair.getValue().getDouble()); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - if (timeValuePair.getValue().getDataType() == TSDataType.DATE) { - builder - .getColumnBuilder(0) - .writeBinary( - new Binary( - TSDataType.getDateStringValue(timeValuePair.getValue().getInt()), - StandardCharsets.UTF_8)); + TsPrimitiveType primitiveType = timeValuePair.getValue(); + if (dataType == TSDataType.VECTOR) { + TsPrimitiveType[] values = timeValuePair.getValue().getVector(); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + builder.getColumnBuilder(i).appendNull(); } else { - builder.getColumnBuilder(0).writeBinary(timeValuePair.getValue().getBinary()); + builder.getColumnBuilder(i).writeTsPrimitiveType(values[i]); } - break; - case VECTOR: - TsPrimitiveType[] values = timeValuePair.getValue().getVector(); - for (int i = 0; i < values.length; i++) { - if (values[i] == null) { - builder.getColumnBuilder(i).appendNull(); - } else { - builder.getColumnBuilder(i).writeTsPrimitiveType(values[i]); - } - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); + } + } else { + builder.getColumnBuilder(0).writeTsPrimitiveType(primitiveType); } builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java index dc95221f0a95..3c1e19a80d80 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java @@ -38,11 +38,11 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.util.ArrayList; import java.util.List; @@ -742,30 +742,8 @@ String getNthIdColumnValue(DeviceEntry deviceEntry, int idColumnIndex) { } private TsPrimitiveType cloneTsPrimitiveType(TsPrimitiveType originalValue) { - switch (originalValue.getDataType()) { - case BOOLEAN: - return new TsPrimitiveType.TsBoolean(originalValue.getBoolean()); - case INT32: - case DATE: - return new TsPrimitiveType.TsInt(originalValue.getInt()); - case INT64: - case TIMESTAMP: - return new TsPrimitiveType.TsLong(originalValue.getLong()); - case FLOAT: - return new TsPrimitiveType.TsFloat(originalValue.getFloat()); - case DOUBLE: - return new TsPrimitiveType.TsDouble(originalValue.getDouble()); - case TEXT: - case BLOB: - case OBJECT: - case STRING: - return new TsPrimitiveType.TsBinary(originalValue.getBinary()); - case VECTOR: - return new TsPrimitiveType.TsVector(originalValue.getVector()); - default: - throw new UnSupportedDataTypeException( - DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + originalValue.getDataType()); - } + return Type.fromTsDataType(originalValue.getDataType()) + .getTsPrimitiveType(originalValue.getValue()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java index dc79a959b1fa..9f9f515e41de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java @@ -21,8 +21,10 @@ import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.execution.aggregation.timerangeiterator.ITimeRangeIterator; +import org.apache.iotdb.db.utils.TypeServices; +import org.apache.iotdb.db.utils.TypeServices.Aggregation.EventWindowManagerProvider; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.read.common.type.Type; public class WindowManagerFactory { @@ -59,57 +61,19 @@ public static IWindowManager genWindowManager( private static VariationWindowManager genEqualEventWindowManager( VariationWindowParameter eventWindowParameter, boolean ascending) { - switch (eventWindowParameter.getDataType()) { - case INT32: - return new EqualIntWindowManager(eventWindowParameter, ascending); - case INT64: - return new EqualLongWindowManager(eventWindowParameter, ascending); - case FLOAT: - return new EqualFloatWindowManager(eventWindowParameter, ascending); - case DOUBLE: - return new EqualDoubleWindowManager(eventWindowParameter, ascending); - case TEXT: - return new EqualBinaryWindowManager(eventWindowParameter, ascending); - case BOOLEAN: - return new EqualBooleanWindowManager(eventWindowParameter, ascending); - case BLOB: - case OBJECT: - case STRING: - case TIMESTAMP: - case DATE: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EQUAL_EVENT_AGGREGATION_S_5076ACFE, - eventWindowParameter.getDataType())); - } + return getEventWindowManagerProvider(eventWindowParameter) + .createEqual(eventWindowParameter, ascending); } private static VariationWindowManager genVariationEventWindowManager( VariationWindowParameter eventWindowParameter, boolean ascending) { - switch (eventWindowParameter.getDataType()) { - case INT32: - return new VariationIntWindowManager(eventWindowParameter, ascending); - case INT64: - return new VariationLongWindowManager(eventWindowParameter, ascending); - case FLOAT: - return new VariationFloatWindowManager(eventWindowParameter, ascending); - case DOUBLE: - return new VariationDoubleWindowManager(eventWindowParameter, ascending); - case TIMESTAMP: - case DATE: - case STRING: - case BOOLEAN: - case BLOB: - case OBJECT: - case TEXT: - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_VARIATION_EVENT_AGGREGATION_S_47341532, - eventWindowParameter.getDataType())); - } + return getEventWindowManagerProvider(eventWindowParameter) + .createVariation(eventWindowParameter, ascending); + } + + private static EventWindowManagerProvider getEventWindowManagerProvider( + final VariationWindowParameter parameter) { + return TypeServices.Aggregation.EVENT_WINDOW_MANAGER_PROVIDER_SERVICE.call( + Type.fromTsDataType(parameter.getDataType())); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/ExistUnknownTypeInExpression.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/ExistUnknownTypeInExpression.java index d786e48dd008..c02bc2b3f387 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/ExistUnknownTypeInExpression.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/ExistUnknownTypeInExpression.java @@ -51,7 +51,7 @@ public List visitFunctionExpression( @Override public List visitTimeSeriesOperand( TimeSeriesOperand timeSeriesOperand, Void context) { - if (timeSeriesOperand.getPath().getSeriesType() == TSDataType.UNKNOWN) { + if (timeSeriesOperand.getOperandType() == TSDataType.UNKNOWN) { return Collections.singletonList(timeSeriesOperand); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/GetMeasurementExpressionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/GetMeasurementExpressionVisitor.java index 4eefc378cfad..bb04a6a426a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/GetMeasurementExpressionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/GetMeasurementExpressionVisitor.java @@ -26,6 +26,8 @@ import org.apache.iotdb.db.queryengine.plan.expression.Expression; import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand; +import org.apache.tsfile.enums.TSDataType; + public class GetMeasurementExpressionVisitor extends ReconstructVisitor { @Override @@ -49,6 +51,10 @@ public Expression visitTimeSeriesOperand(TimeSeriesOperand timeSeriesOperand, An : rawPath.getMeasurement(); return new TimeSeriesOperand( new PartialPath(measurementName, false), rawPath.getMeasurementSchema().getType()); + } else if (timeSeriesOperand.getType() == TSDataType.UNKNOWN) { + // Missing HAVING measurements still need the same output symbol as on other devices. + return new TimeSeriesOperand( + new PartialPath(timeSeriesOperand.getPath().getMeasurement(), false), TSDataType.UNKNOWN); } else { return new TimeSeriesOperand( new PartialPath(timeSeriesOperand.getPath().getNodes()), timeSeriesOperand.getType()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToFilterVisitor.java index a2ff5ef6adbf..3e2554f6bb8c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToFilterVisitor.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.queryengine.plan.expression.visitor.predicate; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.expression.Expression; @@ -43,18 +42,17 @@ import org.apache.iotdb.db.queryengine.plan.expression.unary.LikeExpression; import org.apache.iotdb.db.queryengine.plan.expression.unary.LogicNotExpression; import org.apache.iotdb.db.queryengine.plan.expression.unary.RegularExpression; +import org.apache.iotdb.db.utils.TypeServices; -import com.google.common.io.BaseEncoding; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.regexp.LikePattern; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.lang3.math.NumberUtils; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.FilterFactory; import org.apache.tsfile.read.filter.factory.ValueFilterApi; import org.apache.tsfile.read.filter.operator.FalseLiteralFilter; import org.apache.tsfile.read.filter.operator.ValueIsNotNullOperator; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.schema.IMeasurementSchema; import java.math.BigDecimal; @@ -491,45 +489,14 @@ private > Filter constructBetweenFilter( @SuppressWarnings("unchecked") public static > T getValue(String valueString, TSDataType dataType) { try { - switch (dataType) { - case INT32: - return (T) Integer.valueOf(valueString); - case INT64: - case TIMESTAMP: - return (T) Long.valueOf(valueString); - case FLOAT: - return (T) Float.valueOf(valueString); - case DOUBLE: - return (T) Double.valueOf(valueString); - case BOOLEAN: - if (valueString.equalsIgnoreCase("true")) { - return (T) Boolean.TRUE; - } else if (valueString.equalsIgnoreCase("false")) { - return (T) Boolean.FALSE; - } else { - throw new IllegalArgumentException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_S_CANNOT_BE_CAST_TO_S_DABC2DA0, - valueString, - dataType)); - } - case BLOB: - return (T) new Binary(BaseEncoding.base16().decode(valueString)); - case TEXT: - case STRING: - return (T) new Binary(valueString, TSFileConfig.STRING_CHARSET); - case DATE: - return (T) DateTimeUtils.parseDateExpressionToInt(valueString); - default: - throw new UnsupportedOperationException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_S_4CB21D47, - dataType)); - } + return (T) + TypeServices.Predicate.CONVERT_PREDICATE_VALUE_PARSER_SERVICE + .call(Type.fromTsDataType(dataType)) + .apply(valueString); } catch (NumberFormatException e) { throw new IllegalArgumentException( String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_S_CANNOT_BE_CAST_TO_S_DABC2DA0, + DataNodeQueryMessages.EXCEPTION_ARG_CANNOT_BE_CAST_TO_ARG_28F0C5FC, valueString, dataType)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorGeneratorUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorGeneratorUtil.java index 2d8a35c25474..057a113ce035 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorGeneratorUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorGeneratorUtil.java @@ -20,16 +20,15 @@ package org.apache.iotdb.db.queryengine.plan.planner; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.queryengine.statistics.StatisticsManager; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static org.apache.iotdb.calc.plan.planner.CommonOperatorUtils.UNKNOWN_DATATYPE; - public class OperatorGeneratorUtil { public static long calculateStatementSizePerLine(List dataTypeList) { @@ -54,25 +53,8 @@ public static long calculateStatementSizePerLine( } private static long getValueSizePerLine(TSDataType tsDataType) { - switch (tsDataType) { - case INT32: - case DATE: - return Integer.BYTES; - case INT64: - case TIMESTAMP: - return Long.BYTES; - case FLOAT: - return Float.BYTES; - case DOUBLE: - return Double.BYTES; - case BOOLEAN: - return Byte.BYTES; - case TEXT: - case BLOB: - case STRING: - return StatisticsManager.getInstance().getMaxBinarySizeInBytes(); - default: - throw new UnsupportedOperationException(UNKNOWN_DATATYPE + tsDataType); - } + return TypeServices.Memory.STATEMENT_VALUE_SIZE_PER_LINE_SERVICE + .call(Type.fromTsDataType(tsDataType)) + .getAsLong(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index bedfe1aeff16..ce2f698e5f51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -29,12 +29,6 @@ import org.apache.iotdb.calc.execution.operator.process.OffsetOperator; import org.apache.iotdb.calc.execution.operator.process.ProcessOperator; import org.apache.iotdb.calc.execution.operator.process.fill.IFill; -import org.apache.iotdb.calc.execution.operator.process.fill.constant.BinaryConstantFill; -import org.apache.iotdb.calc.execution.operator.process.fill.constant.BooleanConstantFill; -import org.apache.iotdb.calc.execution.operator.process.fill.constant.DoubleConstantFill; -import org.apache.iotdb.calc.execution.operator.process.fill.constant.FloatConstantFill; -import org.apache.iotdb.calc.execution.operator.process.fill.constant.IntConstantFill; -import org.apache.iotdb.calc.execution.operator.process.fill.constant.LongConstantFill; import org.apache.iotdb.calc.plan.planner.CommonOperatorUtils; import org.apache.iotdb.calc.plan.planner.memory.PipelineMemoryEstimator; import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; @@ -263,6 +257,7 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.operator.TimeFilterOperators.TimeGt; import org.apache.tsfile.read.filter.operator.TimeFilterOperators.TimeGtEq; @@ -305,6 +300,7 @@ import static org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.AggregationDescriptor.getAggregationTypeByFuncName; import static org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions.updateFilterUsingTTL; import static org.apache.iotdb.db.queryengine.plan.statement.component.Ordering.ASC; +import static org.apache.iotdb.db.utils.TypeServices.Transformation.CONSTANT_FILL_SERVICE; /** This Visitor is responsible for transferring PlanNode Tree to Operator Tree. */ public class OperatorTreeGenerator implements PlanVisitor { @@ -1359,35 +1355,8 @@ private IFill[] getConstantFill( constantFill[i] = CommonOperatorUtils.IDENTITY_FILL; continue; } - switch (inputDataTypes.get(i)) { - case BOOLEAN: - constantFill[i] = new BooleanConstantFill(literal.getBoolean()); - break; - case TEXT: - case BLOB: - case STRING: - constantFill[i] = new BinaryConstantFill(literal.getBinary()); - break; - case INT32: - constantFill[i] = new IntConstantFill(literal.getInt()); - break; - case DATE: - constantFill[i] = new IntConstantFill(literal.getDate()); - break; - case INT64: - case TIMESTAMP: - constantFill[i] = new LongConstantFill(literal.getLong()); - break; - case FLOAT: - constantFill[i] = new FloatConstantFill(literal.getFloat()); - break; - case DOUBLE: - constantFill[i] = new DoubleConstantFill(literal.getDouble()); - break; - default: - throw new IllegalArgumentException( - CommonOperatorUtils.UNKNOWN_DATATYPE + inputDataTypes.get(i)); - } + constantFill[i] = + CONSTANT_FILL_SERVICE.call(Type.fromTsDataType(inputDataTypes.get(i))).apply(literal); } return constantFill; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index d7a09a6b49d7..ecf76892e4cd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -41,13 +41,13 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.iotdb.db.utils.TypeInferenceUtils; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NotImplementedException; import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -399,16 +399,9 @@ private int serializedValueSize(final int index) { } return Byte.BYTES - + switch (dataType) { - case BOOLEAN -> Byte.BYTES; - case INT32, DATE -> Integer.BYTES; - case INT64, TIMESTAMP -> Long.BYTES; - case FLOAT -> Float.BYTES; - case DOUBLE -> Double.BYTES; - case TEXT, STRING, BLOB, OBJECT -> ReadWriteIOUtils.sizeToWrite((Binary) values[index]); - case VECTOR, UNKNOWN -> - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); - }; + + TypeServices.StorageEngine.INSERT_ROW_SERIALIZED_VALUE_SIZE_SERVICE + .call(Type.fromTsDataType(dataType)) + .applyAsInt(values[index]); } void subSerialize(ByteBuffer buffer) { @@ -514,33 +507,7 @@ private void putDataTypesAndValues(ByteBuffer buffer) { ReadWriteIOUtils.write(values[i].toString(), buffer); } else { ReadWriteIOUtils.write(dataType, buffer); - switch (dataType) { - case BOOLEAN: - ReadWriteIOUtils.write((Boolean) values[i], buffer); - break; - case INT32: - case DATE: - ReadWriteIOUtils.write((Integer) values[i], buffer); - break; - case INT64: - case TIMESTAMP: - ReadWriteIOUtils.write((Long) values[i], buffer); - break; - case FLOAT: - ReadWriteIOUtils.write((Float) values[i], buffer); - break; - case DOUBLE: - ReadWriteIOUtils.write((Double) values[i], buffer); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - ReadWriteIOUtils.write((Binary) values[i], buffer); - break; - default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); - } + Type.fromTsDataType(dataType).serializeValue(values[i], buffer); } } } @@ -575,33 +542,7 @@ private void putDataTypesAndValues(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(values[i].toString(), stream); } else { ReadWriteIOUtils.write(dataType, stream); - switch (dataType) { - case BOOLEAN: - ReadWriteIOUtils.write((Boolean) values[i], stream); - break; - case INT32: - case DATE: - ReadWriteIOUtils.write((Integer) values[i], stream); - break; - case INT64: - case TIMESTAMP: - ReadWriteIOUtils.write((Long) values[i], stream); - break; - case FLOAT: - ReadWriteIOUtils.write((Float) values[i], stream); - break; - case DOUBLE: - ReadWriteIOUtils.write((Double) values[i], stream); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - ReadWriteIOUtils.write((Binary) values[i], stream); - break; - default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); - } + Type.fromTsDataType(dataType).serializeValue(values[i], stream); } } } @@ -670,33 +611,7 @@ private void fillDataTypesAndValues(ByteBuffer buffer) { continue; } dataTypes[i] = TSDataType.values()[typeNum]; - switch (dataTypes[i]) { - case BOOLEAN: - values[i] = ReadWriteIOUtils.readBool(buffer); - break; - case INT32: - case DATE: - values[i] = ReadWriteIOUtils.readInt(buffer); - break; - case INT64: - case TIMESTAMP: - values[i] = ReadWriteIOUtils.readLong(buffer); - break; - case FLOAT: - values[i] = ReadWriteIOUtils.readFloat(buffer); - break; - case DOUBLE: - values[i] = ReadWriteIOUtils.readDouble(buffer); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - values[i] = ReadWriteIOUtils.readBinary(buffer); - break; - default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataTypes[i]); - } + Type.fromTsDataType(dataTypes[i]).deserialize(values, i, buffer); } } @@ -741,32 +656,8 @@ private int serializeMeasurementsAndValuesSize() { continue; } size += Byte.BYTES; - switch (dataType) { - case BOOLEAN: - size += Byte.BYTES; - break; - case INT32: - case DATE: - size += Integer.BYTES; - break; - case INT64: - case TIMESTAMP: - size += Long.BYTES; - break; - case FLOAT: - size += Float.BYTES; - break; - case DOUBLE: - size += Double.BYTES; - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - size += ReadWriteIOUtils.sizeToWrite((Binary) values[i]); - break; - default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); + if (dataType != null) { + size += Type.fromTsDataType(dataType).calcTypeSize(values[i]); } } @@ -826,33 +717,9 @@ private void putDataTypesAndValues(IWALByteBufferView buffer) { continue; } WALWriteUtils.write(dataType, buffer); - switch (dataType) { - case BOOLEAN: - WALWriteUtils.write((Boolean) values[i], buffer); - break; - case INT32: - case DATE: - WALWriteUtils.write((Integer) values[i], buffer); - break; - case INT64: - case TIMESTAMP: - WALWriteUtils.write((Long) values[i], buffer); - break; - case FLOAT: - WALWriteUtils.write((Float) values[i], buffer); - break; - case DOUBLE: - WALWriteUtils.write((Double) values[i], buffer); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - WALWriteUtils.write((Binary) values[i], buffer); - break; - default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); - } + TypeServices.StorageEngine.WAL_VALUE_WRITER_SERVICE + .call(Type.fromTsDataType(dataType)) + .accept(values[i], buffer); } } @@ -918,33 +785,7 @@ public void fillDataTypesAndValuesFromWAL(DataInputStream stream) throws IOExcep continue; } dataTypes[i] = TSDataType.values()[typeNum]; - switch (dataTypes[i]) { - case BOOLEAN: - values[i] = ReadWriteIOUtils.readBool(stream); - break; - case INT32: - case DATE: - values[i] = ReadWriteIOUtils.readInt(stream); - break; - case INT64: - case TIMESTAMP: - values[i] = ReadWriteIOUtils.readLong(stream); - break; - case FLOAT: - values[i] = ReadWriteIOUtils.readFloat(stream); - break; - case DOUBLE: - values[i] = ReadWriteIOUtils.readDouble(stream); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - values[i] = ReadWriteIOUtils.readBinary(stream); - break; - default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataTypes[i]); - } + Type.fromTsDataType(dataTypes[i]).deserialize(values, i, stream); } } @@ -1009,33 +850,7 @@ public void fillDataTypesAndValuesFromWAL(ByteBuffer buffer) { continue; } dataTypes[i] = TSDataType.values()[typeNum]; - switch (dataTypes[i]) { - case BOOLEAN: - values[i] = ReadWriteIOUtils.readBool(buffer); - break; - case INT32: - case DATE: - values[i] = ReadWriteIOUtils.readInt(buffer); - break; - case INT64: - case TIMESTAMP: - values[i] = ReadWriteIOUtils.readLong(buffer); - break; - case FLOAT: - values[i] = ReadWriteIOUtils.readFloat(buffer); - break; - case DOUBLE: - values[i] = ReadWriteIOUtils.readDouble(buffer); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - values[i] = ReadWriteIOUtils.readBinary(buffer); - break; - default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataTypes[i]); - } + Type.fromTsDataType(dataTypes[i]).deserialize(values, i, buffer); } } @@ -1075,7 +890,8 @@ public TimeValuePair composeTimeValuePair(int columnIndex) { return null; } Object value = values[columnIndex]; - return new TimeValuePair(time, TsPrimitiveType.getByType(dataTypes[columnIndex], value)); + return new TimeValuePair( + time, Type.fromTsDataType(dataTypes[columnIndex]).getTsPrimitiveType(value)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index 91c1a6d1c308..e0b1f0b4205e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.iotdb.db.utils.BitMapUtils; import org.apache.iotdb.db.utils.QueryDataSetUtils; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -53,13 +54,11 @@ import org.apache.tsfile.exception.NotImplementedException; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.schema.MeasurementSchema; import java.io.DataInputStream; @@ -80,7 +79,6 @@ import static org.apache.iotdb.db.utils.CommonUtils.isAlive; public class InsertTabletNode extends InsertNode implements WALEntryValue { - private static final String DATATYPE_UNSUPPORTED = "Data type %s is not supported."; protected long[] times; // times should be sorted. It is done in the session API. @@ -420,31 +418,10 @@ private Object[] initTabletValues( || forSplit && !hasColumnForSplit(i)) { continue; } - switch (dataTypes[i]) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - values[i] = new Binary[rowSize]; - break; - case FLOAT: - values[i] = new float[rowSize]; - break; - case INT32: - case DATE: - values[i] = new int[rowSize]; - break; - case TIMESTAMP: - case INT64: - values[i] = new long[rowSize]; - break; - case DOUBLE: - values[i] = new double[rowSize]; - break; - case BOOLEAN: - values[i] = new boolean[rowSize]; - break; - } + values[i] = + TypeServices.StorageEngine.PRIMITIVE_ARRAY_ALLOCATOR_SERVICE + .call(Type.fromTsDataType(dataTypes[i])) + .apply(rowSize); } return values; } @@ -631,26 +608,9 @@ final int serializedSubAttributesSize() { } private int serializedColumnSize(final TSDataType dataType, final Object column) { - return switch (dataType) { - case BOOLEAN -> rowCount * Byte.BYTES; - case INT32, DATE -> rowCount * Integer.BYTES; - case INT64, TIMESTAMP -> rowCount * Long.BYTES; - case FLOAT -> rowCount * Float.BYTES; - case DOUBLE -> rowCount * Double.BYTES; - case TEXT, BLOB, STRING, OBJECT -> serializedBinaryColumnSize((Binary[]) column); - case VECTOR, UNKNOWN -> - throw new UnSupportedDataTypeException(String.format(DATATYPE_UNSUPPORTED, dataType)); - }; - } - - private int serializedBinaryColumnSize(final Binary[] binaryValues) { - int size = 0; - for (int i = 0; i < rowCount; i++) { - final Binary binary = binaryValues[i]; - final byte[] values = binary == null ? null : binary.getValues(); - size += values == null ? Integer.BYTES : Integer.BYTES + values.length; - } - return size; + return TypeServices.StorageEngine.INSERT_TABLET_SERIALIZED_COLUMN_SIZE_SERVICE + .call(Type.fromTsDataType(dataType)) + .size(column, rowCount); } /** Serialize measurements or measurement schemas, ignoring failed time series */ @@ -792,108 +752,16 @@ private void writeValues(DataOutputStream stream) throws IOException { } private void serializeColumn(TSDataType dataType, Object column, ByteBuffer buffer) { - switch (dataType) { - case INT32: - case DATE: - int[] intValues = (int[]) column; - for (int j = 0; j < rowCount; j++) { - ReadWriteIOUtils.write(intValues[j], buffer); - } - break; - case INT64: - case TIMESTAMP: - long[] longValues = (long[]) column; - for (int j = 0; j < rowCount; j++) { - ReadWriteIOUtils.write(longValues[j], buffer); - } - break; - case FLOAT: - float[] floatValues = (float[]) column; - for (int j = 0; j < rowCount; j++) { - ReadWriteIOUtils.write(floatValues[j], buffer); - } - break; - case DOUBLE: - double[] doubleValues = (double[]) column; - for (int j = 0; j < rowCount; j++) { - ReadWriteIOUtils.write(doubleValues[j], buffer); - } - break; - case BOOLEAN: - boolean[] boolValues = (boolean[]) column; - for (int j = 0; j < rowCount; j++) { - ReadWriteIOUtils.write(BytesUtils.boolToByte(boolValues[j]), buffer); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary[] binaryValues = (Binary[]) column; - for (int j = 0; j < rowCount; j++) { - if (binaryValues[j] != null && binaryValues[j].getValues() != null) { - ReadWriteIOUtils.write(binaryValues[j], buffer); - } else { - ReadWriteIOUtils.write(0, buffer); - } - } - break; - default: - throw new UnSupportedDataTypeException(String.format(DATATYPE_UNSUPPORTED, dataType)); - } + TypeServices.StorageEngine.RAW_ARRAY_BYTE_BUFFER_SERIALIZER_SERVICE + .call(Type.fromTsDataType(dataType)) + .serialize(column, rowCount, buffer); } private void serializeColumn(TSDataType dataType, Object column, DataOutputStream stream) throws IOException { - switch (dataType) { - case INT32: - case DATE: - int[] intValues = (int[]) column; - for (int j = 0; j < rowCount; j++) { - ReadWriteIOUtils.write(intValues[j], stream); - } - break; - case INT64: - case TIMESTAMP: - long[] longValues = (long[]) column; - for (int j = 0; j < rowCount; j++) { - ReadWriteIOUtils.write(longValues[j], stream); - } - break; - case FLOAT: - float[] floatValues = (float[]) column; - for (int j = 0; j < rowCount; j++) { - ReadWriteIOUtils.write(floatValues[j], stream); - } - break; - case DOUBLE: - double[] doubleValues = (double[]) column; - for (int j = 0; j < rowCount; j++) { - ReadWriteIOUtils.write(doubleValues[j], stream); - } - break; - case BOOLEAN: - boolean[] boolValues = (boolean[]) column; - for (int j = 0; j < rowCount; j++) { - ReadWriteIOUtils.write(BytesUtils.boolToByte(boolValues[j]), stream); - } - break; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - Binary[] binaryValues = (Binary[]) column; - for (int j = 0; j < rowCount; j++) { - if (binaryValues[j] != null && binaryValues[j].getValues() != null) { - ReadWriteIOUtils.write(binaryValues[j], stream); - } else { - ReadWriteIOUtils.write(0, stream); - } - } - break; - default: - throw new UnSupportedDataTypeException(String.format(DATATYPE_UNSUPPORTED, dataType)); - } + TypeServices.StorageEngine.RAW_ARRAY_OUTPUT_STREAM_SERIALIZER_SERVICE + .call(Type.fromTsDataType(dataType)) + .serialize(column, rowCount, stream); } public static InsertTabletNode deserialize(ByteBuffer byteBuffer) { @@ -1032,36 +900,9 @@ private int getRowNumInRange(List rangeList) { } private int getColumnSize(TSDataType dataType, Object column, int start, int end) { - int size = 0; - switch (dataType) { - case INT32: - case DATE: - size += Integer.BYTES * (end - start); - break; - case INT64: - case TIMESTAMP: - size += Long.BYTES * (end - start); - break; - case FLOAT: - size += Float.BYTES * (end - start); - break; - case DOUBLE: - size += Double.BYTES * (end - start); - break; - case BOOLEAN: - size += Byte.BYTES * (end - start); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary[] binaryValues = (Binary[]) column; - for (int j = start; j < end; j++) { - size += ReadWriteIOUtils.sizeToWrite(binaryValues[j]); - } - break; - } - return size; + return TypeServices.StorageEngine.INSERT_TABLET_SERIALIZED_COLUMN_SIZE_SERVICE + .call(Type.fromTsDataType(dataType)) + .size(column, start, end); } /** @@ -1235,57 +1076,10 @@ protected boolean shouldSerializeMeasurementToWAL(int index) { } private void serializeColumn( - TSDataType dataType, Object column, IWALByteBufferView buffer, int start, int end) { - switch (dataType) { - case INT32: - case DATE: - int[] intValues = (int[]) column; - for (int j = start; j < end; j++) { - buffer.putInt(intValues[j]); - } - break; - case INT64: - case TIMESTAMP: - long[] longValues = (long[]) column; - for (int j = start; j < end; j++) { - buffer.putLong(longValues[j]); - } - break; - case FLOAT: - float[] floatValues = (float[]) column; - for (int j = start; j < end; j++) { - buffer.putFloat(floatValues[j]); - } - break; - case DOUBLE: - double[] doubleValues = (double[]) column; - for (int j = start; j < end; j++) { - buffer.putDouble(doubleValues[j]); - } - break; - case BOOLEAN: - boolean[] boolValues = (boolean[]) column; - for (int j = start; j < end; j++) { - buffer.put(BytesUtils.boolToByte(boolValues[j])); - } - break; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - Binary[] binaryValues = (Binary[]) column; - for (int j = start; j < end; j++) { - if (binaryValues[j] != null && binaryValues[j].getValues() != null) { - buffer.putInt(binaryValues[j].getLength()); - buffer.put(binaryValues[j].getValues()); - } else { - buffer.putInt(0); - } - } - break; - default: - throw new UnSupportedDataTypeException(String.format(DATATYPE_UNSUPPORTED, dataType)); - } + TSDataType dataType, Object array, IWALByteBufferView buffer, int start, int end) { + TypeServices.StorageEngine.WAL_ARRAY_WRITER_SERVICE + .call(Type.fromTsDataType(dataType)) + .write(array, buffer, start, end); } /** Deserialize from wal */ @@ -1408,44 +1202,8 @@ private boolean equals(Object[] columns) { for (int i = 0; i < columns.length; i++) { final TSDataType dataType = getDataType(i); if (dataType != null) { - switch (dataType) { - case INT32: - case DATE: - if (!Arrays.equals((int[]) this.columns[i], (int[]) columns[i])) { - return false; - } - break; - case INT64: - case TIMESTAMP: - if (!Arrays.equals((long[]) this.columns[i], (long[]) columns[i])) { - return false; - } - break; - case FLOAT: - if (!Arrays.equals((float[]) this.columns[i], (float[]) columns[i])) { - return false; - } - break; - case DOUBLE: - if (!Arrays.equals((double[]) this.columns[i], (double[]) columns[i])) { - return false; - } - break; - case BOOLEAN: - if (!Arrays.equals((boolean[]) this.columns[i], (boolean[]) columns[i])) { - return false; - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - if (!Arrays.equals((Binary[]) this.columns[i], (Binary[]) columns[i])) { - return false; - } - break; - default: - throw new UnSupportedDataTypeException(String.format(DATATYPE_UNSUPPORTED, dataType)); + if (!Type.fromTsDataType(dataType).arrayEquals(this.columns[i], columns[i], rowCount)) { + return false; } } else if (!Objects.equals(this.columns[i], columns[i])) { return false; @@ -1537,43 +1295,14 @@ private BitMap getBitMapIfPresent(final int measurementIndex) { } private TimeValuePair composeTimeValuePair(final int measurementIndex, final int rowIndex) { - TsPrimitiveType value; - switch (dataTypes[measurementIndex]) { - case INT32: - case DATE: - int[] intValues = (int[]) columns[measurementIndex]; - value = new TsPrimitiveType.TsInt(intValues[rowIndex]); - break; - case INT64: - case TIMESTAMP: - long[] longValues = (long[]) columns[measurementIndex]; - value = new TsPrimitiveType.TsLong(longValues[rowIndex]); - break; - case FLOAT: - float[] floatValues = (float[]) columns[measurementIndex]; - value = new TsPrimitiveType.TsFloat(floatValues[rowIndex]); - break; - case DOUBLE: - double[] doubleValues = (double[]) columns[measurementIndex]; - value = new TsPrimitiveType.TsDouble(doubleValues[rowIndex]); - break; - case BOOLEAN: - boolean[] boolValues = (boolean[]) columns[measurementIndex]; - value = new TsPrimitiveType.TsBoolean(boolValues[rowIndex]); - break; - case TEXT: - case BLOB: - case STRING: - Binary[] binaryValues = (Binary[]) columns[measurementIndex]; - value = new TsPrimitiveType.TsBinary(binaryValues[rowIndex]); - break; - case OBJECT: - return null; - default: - throw new UnSupportedDataTypeException( - String.format(DATATYPE_UNSUPPORTED, dataTypes[measurementIndex])); + final TSDataType dataType = dataTypes[measurementIndex]; + if (dataType == TSDataType.OBJECT) { + return null; } - return new TimeValuePair(times[rowIndex], value); + return new TimeValuePair( + times[rowIndex], + Type.fromTsDataType(dataType) + .getValueAsTsPrimitiveType(columns[measurementIndex], rowIndex)); } public IDeviceID getDeviceID(int rowIdx) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java index 80ddbb8cddf2..ecd45a638b7b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java @@ -19,18 +19,14 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate; -import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.commons.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BetweenPredicate; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BinaryLiteral; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FloatLiteral; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.GenericLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IfExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; @@ -49,10 +45,10 @@ import org.apache.iotdb.commons.queryengine.plan.relational.type.InternalTypeManager; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.utils.TypeServices; import com.google.common.collect.ImmutableList; import com.google.common.math.DoubleMath; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.regexp.LikePattern; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.type.LongType; @@ -63,7 +59,6 @@ import org.apache.tsfile.read.filter.operator.ExtractTimeFilterOperators; import org.apache.tsfile.read.filter.operator.FalseLiteralFilter; import org.apache.tsfile.read.filter.operator.ValueIsNotNullOperator; -import org.apache.tsfile.utils.Binary; import javax.annotation.Nullable; @@ -363,38 +358,14 @@ private Filter constructExtractCompareFilter( @SuppressWarnings("unchecked") public static > T getValue(Literal value, Type dataType) { try { - switch (dataType.getTypeEnum()) { - case INT32: - return (T) Integer.valueOf((int) getLongValue(value)); - case DATE: - return (T) getDateValue(value); - case INT64: - return (T) Long.valueOf(getLongValue(value)); - case TIMESTAMP: - return (T) getTimestampValue(value); - case FLOAT: - return (T) Float.valueOf((float) getDoubleValue(value)); - case DOUBLE: - return (T) Double.valueOf(getDoubleValue(value)); - case BOOLEAN: - return (T) Boolean.valueOf(getBooleanValue(value)); - case TEXT: - case STRING: - return (T) new Binary(getStringValue(value), TSFileConfig.STRING_CHARSET); - case BLOB: - return (T) new Binary(getBlobValue(value)); - default: - throw new UnsupportedOperationException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_S_4CB21D47, - dataType)); - } + return (T) + TypeServices.Predicate.RELATIONAL_CONVERT_PREDICATE_VALUE_PARSER_SERVICE + .call(dataType) + .apply(value); } catch (NumberFormatException e) { throw new IllegalArgumentException( String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_S_CANNOT_BE_CAST_TO_S_DABC2DA0, - value, - dataType)); + DataNodeQueryMessages.EXCEPTION_ARG_CANNOT_BE_CAST_TO_ARG_28F0C5FC, value, dataType)); } } @@ -611,50 +582,6 @@ private > Filter constructBetweenFilter( return ValueFilterApi.between(measurementIndex, min, max, dataType); } - public static double getDoubleValue(Expression expression) { - if (expression instanceof DoubleLiteral) { - return ((DoubleLiteral) expression).getValue(); - } else if (expression instanceof LongLiteral) { - return ((LongLiteral) expression).getParsedValue(); - } else if (expression instanceof FloatLiteral) { - return ((FloatLiteral) expression).getValue(); - } else { - throw new IllegalArgumentException( - DataNodeQueryMessages.EXPRESSION_SHOULD_BE_NUMERIC_ACTUAL_IS + expression); - } - } - - public static boolean getBooleanValue(Expression expression) { - return ((BooleanLiteral) expression).getValue(); - } - - public static String getStringValue(Expression expression) { - return ((StringLiteral) expression).getValue(); - } - - public static byte[] getBlobValue(Expression expression) { - return ((BinaryLiteral) expression).getValue(); - } - - public static Integer getDateValue(Expression expression) { - return Integer.valueOf(((GenericLiteral) expression).getValue()); - } - - public static Long getTimestampValue(Expression expression) { - if (expression instanceof LongLiteral) { - return ((LongLiteral) expression).getParsedValue(); - } else if (expression instanceof DoubleLiteral) { - return (long) ((DoubleLiteral) expression).getValue(); - } else if (expression instanceof GenericLiteral) { - return Long.valueOf(((GenericLiteral) expression).getValue()); - } else { - throw new SemanticException( - DataNodeQueryMessages - .INLIST_LITERAL_FOR_TIMESTAMP_CAN_ONLY_BE_LONGLITERAL_DOUBLELITERAL_AND_GENERICLITERAL - + expression.getClass().getSimpleName()); - } - } - public static class Context { private final Map measuremrntsMap; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java index 7789de35f3a4..82d8113a2f52 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java @@ -20,21 +20,15 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.ir; import org.apache.iotdb.commons.exception.IoTDBException; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BinaryLiteral; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DereferenceExpression; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FloatLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Literal; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Query; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.SubqueryExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.With; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.WithQuery; @@ -49,11 +43,13 @@ import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.Type; import java.util.List; import java.util.Optional; @@ -207,33 +203,11 @@ public Optional fetchUncorrelatedSubqueryResultForPredicate( } checkArgument( dataType != null && column != null, - DataNodeQueryMessages - .EXCEPTION_SCALAR_SUBQUERY_RESULT_SHOULD_NOT_GET_NULL_DATATYPE_OR_NULL_COLUMN_DOT_616056F4); - switch (dataType) { - case INT32: - case DATE: - return Optional.of(new LongLiteral(Long.toString(column.getInt(0)))); - case INT64: - case TIMESTAMP: - return Optional.of(new LongLiteral(Long.toString(column.getLong(0)))); - case FLOAT: - return Optional.of(new FloatLiteral(column.getFloat(0))); - case DOUBLE: - return Optional.of(new DoubleLiteral(Double.toString(column.getDouble(0)))); - case BOOLEAN: - return Optional.of(new BooleanLiteral(Boolean.toString(column.getBoolean(0)))); - case BLOB: - return Optional.of(new BinaryLiteral(column.getBinary(0).toString())); - case TEXT: - case STRING: - return Optional.of(new StringLiteral(column.getBinary(0).toString())); - default: - throw new IllegalArgumentException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_UNSUPPORTED_DATA_TYPE_FOR_SCALAR_SUBQUERY_RESULT_S_4381E489, - column.getDataType())); - } + "Scalar Subquery result should not get null dataType or null column."); + return Optional.of( + TypeServices.Predicate.UNCORRELATED_SCALAR_SUBQUERY_RESULT_LITERAL_SERVICE + .call(Type.fromTsDataType(dataType)) + .apply(column)); } catch (final Throwable throwable) { t = throwable; } finally { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java index ecf25ac991f2..0c33daa295f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java @@ -50,6 +50,7 @@ import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; @@ -161,33 +162,7 @@ public void fillValues(ByteBuffer buffer) throws QueryProcessException { continue; } dataTypes[i] = TSDataType.values()[typeNum]; - switch (dataTypes[i]) { - case BOOLEAN: - values[i] = ReadWriteIOUtils.readBool(buffer); - break; - case INT32: - case DATE: - values[i] = ReadWriteIOUtils.readInt(buffer); - break; - case INT64: - case TIMESTAMP: - values[i] = ReadWriteIOUtils.readLong(buffer); - break; - case FLOAT: - values[i] = ReadWriteIOUtils.readFloat(buffer); - break; - case DOUBLE: - values[i] = ReadWriteIOUtils.readDouble(buffer); - break; - case TEXT: - case BLOB: - case STRING: - values[i] = ReadWriteIOUtils.readBinary(buffer); - break; - default: - throw new QueryProcessException( - DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + dataTypes[i]); - } + Type.fromTsDataType(dataTypes[i]).deserialize(values, i, buffer); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index d516bfe45114..7f9d7ce5551a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -52,12 +52,12 @@ import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -79,8 +79,6 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(InsertTabletStatement.class); - private static final String DATATYPE_UNSUPPORTED = "Data type %s is not supported."; - /** * Get the instance size of InsertTabletStatement for memory calculation. * @@ -159,32 +157,11 @@ private Object convertTableColumn(final Object input, final int rowCount, final .map(binary -> Objects.nonNull(binary) ? binary : Binary.EMPTY_VALUE) .toArray(Binary[]::new); } else if (input == null) { - switch (type) { - case BOOLEAN: - return new boolean[rowCount]; - case INT32: - case DATE: - return new int[rowCount]; - case INT64: - case TIMESTAMP: - return new long[rowCount]; - case FLOAT: - return new float[rowCount]; - case DOUBLE: - return new double[rowCount]; - case TEXT: - case BLOB: - case STRING: - final Binary[] result = new Binary[rowCount]; - Arrays.fill(result, Binary.EMPTY_VALUE); - return result; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_WHEN_CONVERT_DATA_AT_CLIENT_405429CC, - type)); + final Object result = Type.fromTsDataType(type).createArray(rowCount); + if (result instanceof Binary[]) { + Arrays.fill((Binary[]) result, Binary.EMPTY_VALUE); } + return result; } return input; @@ -483,41 +460,7 @@ public Object getFirstValueOfIndex(int index) { || columns[index] == null) { return null; } - Object value; - switch (dataTypes[index]) { - case INT32: - case DATE: - int[] intValues = (int[]) columns[index]; - value = intValues[0]; - break; - case INT64: - case TIMESTAMP: - long[] longValues = (long[]) columns[index]; - value = longValues[0]; - break; - case FLOAT: - float[] floatValues = (float[]) columns[index]; - value = floatValues[0]; - break; - case DOUBLE: - double[] doubleValues = (double[]) columns[index]; - value = doubleValues[0]; - break; - case BOOLEAN: - boolean[] boolValues = (boolean[]) columns[index]; - value = boolValues[0]; - break; - case TEXT: - case BLOB: - case STRING: - Binary[] binaryValues = (Binary[]) columns[index]; - value = binaryValues[0]; - break; - default: - throw new UnSupportedDataTypeException( - String.format(DATATYPE_UNSUPPORTED, dataTypes[index])); - } - return value; + return Type.fromTsDataType(dataTypes[index]).getValue(columns[index], 0); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnRow.java index 6f42b95a6acc..e2f16d6f7481 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnRow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnRow.java @@ -62,7 +62,7 @@ public float getFloat(int columnIndex) { @Override public double getDouble(int columnIndex) { - return (double) rowRecord[columnIndex]; + return ((Number) rowRecord[columnIndex]).doubleValue(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java index 55096612ad75..2c8e3689a232 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java @@ -25,21 +25,16 @@ import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.utils.CommonUtils; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.block.column.BinaryColumn; -import org.apache.tsfile.read.common.block.column.BooleanColumn; -import org.apache.tsfile.read.common.block.column.DoubleColumn; -import org.apache.tsfile.read.common.block.column.FloatColumn; -import org.apache.tsfile.read.common.block.column.IntColumn; -import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.IOException; -import java.util.Optional; public class ConstantInputReader implements LayerReader { private final TSDataType dataType; @@ -54,54 +49,22 @@ public ConstantInputReader(ConstantOperand expression) throws QueryProcessExcept Object value = CommonUtils.parseValue(expression.getDataType(), expression.getValueString()); if (value == null) { throw new QueryProcessException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_INVALID_CONSTANT_OPERAND_S_939F3B8D, - expression.getExpressionString())); + DataNodeQueryMessages.UNSUPPORTED_CONSTANT_OPERAND + expression.getExpressionString()); } // Use RLEColumn to mimic column filled with same values dataType = expression.getDataType(); cachedColumns = new Column[1]; int count = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber(); - switch (dataType) { - case INT32: - int[] intArray = {(int) value}; - Column intColumn = new IntColumn(1, Optional.empty(), intArray); - cachedColumns[0] = new RunLengthEncodedColumn(intColumn, count); - break; - case INT64: - long[] longArray = {(long) value}; - Column longColumn = new LongColumn(1, Optional.empty(), longArray); - cachedColumns[0] = new RunLengthEncodedColumn(longColumn, count); - break; - case FLOAT: - float[] floatArray = {(float) value}; - Column floatColumn = new FloatColumn(1, Optional.empty(), floatArray); - cachedColumns[0] = new RunLengthEncodedColumn(floatColumn, count); - break; - case DOUBLE: - double[] doubleArray = {(double) value}; - Column doubleColumn = new DoubleColumn(1, Optional.empty(), doubleArray); - cachedColumns[0] = new RunLengthEncodedColumn(doubleColumn, count); - break; - case TEXT: - Binary[] binaryArray = {(Binary) value}; - Column binaryColumn = new BinaryColumn(1, Optional.empty(), binaryArray); - cachedColumns[0] = new RunLengthEncodedColumn(binaryColumn, count); - break; - case BOOLEAN: - boolean[] booleanArray = {(boolean) value}; - Column booleanColumn = new BooleanColumn(1, Optional.empty(), booleanArray); - cachedColumns[0] = new RunLengthEncodedColumn(booleanColumn, count); - break; - case BLOB: - case OBJECT: - case STRING: - case TIMESTAMP: - case DATE: - default: - throw new QueryProcessException( - DataNodeQueryMessages.UNSUPPORTED_TYPE + expression.getDataType()); + try { + cachedColumns[0] = + new RunLengthEncodedColumn( + TypeServices.Transformation.CONSTANT_COLUMN_BUILDER_SERVICE + .call(Type.fromTsDataType(dataType)) + .apply(value), + count); + } catch (final UnSupportedDataTypeException e) { + throw new QueryProcessException(DataNodeQueryMessages.UNSUPPORTED_TYPE + dataType); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java index d2fd604eb834..84e68738037d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java @@ -26,9 +26,12 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import java.io.IOException; +import static org.apache.iotdb.db.utils.TypeServices.Transformation.NEGATION_TRANSFORMER_SERVICE; + public class ArithmeticNegationTransformer extends UnaryTransformer { public ArithmeticNegationTransformer(LayerReader layerReader) { @@ -43,33 +46,17 @@ public TSDataType[] getDataTypes() { @Override protected void transform(Column[] columns, ColumnBuilder builder) throws QueryProcessException, IOException { - switch (layerReaderDataType) { - case INT32: - transformInt(columns, builder); - return; - case INT64: - transformLong(columns, builder); - return; - case FLOAT: - transformFloat(columns, builder); - return; - case DOUBLE: - transformDouble(columns, builder); - return; - case DATE: - case TEXT: - case TIMESTAMP: - case BLOB: - case OBJECT: - case BOOLEAN: - case STRING: - default: - throw new QueryProcessException( - DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + layerReaderDataType); + final Type type; + try { + type = Type.fromTsDataType(layerReaderDataType); + } catch (UnsupportedOperationException ignored) { + throw new QueryProcessException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + layerReaderDataType); } + NEGATION_TRANSFORMER_SERVICE.call(type).transform(this, columns, builder); } - private void transformInt(Column[] columns, ColumnBuilder builder) { + public void transformInt(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); int[] values = columns[0].getInts(); boolean[] isNulls = columns[0].isNull(); @@ -83,7 +70,7 @@ private void transformInt(Column[] columns, ColumnBuilder builder) { } } - private void transformLong(Column[] columns, ColumnBuilder builder) { + public void transformLong(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); long[] values = columns[0].getLongs(); boolean[] isNulls = columns[0].isNull(); @@ -97,7 +84,7 @@ private void transformLong(Column[] columns, ColumnBuilder builder) { } } - private void transformFloat(Column[] columns, ColumnBuilder builder) { + public void transformFloat(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); float[] values = columns[0].getFloats(); boolean[] isNulls = columns[0].isNull(); @@ -111,7 +98,7 @@ private void transformFloat(Column[] columns, ColumnBuilder builder) { } } - private void transformDouble(Column[] columns, ColumnBuilder builder) { + public void transformDouble(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); double[] values = columns[0].getDoubles(); boolean[] isNulls = columns[0].isNull(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java index 2143ca18488d..1a4a757b15b5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java @@ -20,19 +20,22 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary; import org.apache.iotdb.calc.exception.QueryProcessException; -import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import java.io.IOException; import java.util.HashSet; import java.util.Set; +import static org.apache.iotdb.db.utils.TypeServices.Transformation.IN_TRANSFORMER_COLUMN_TRANSFORMER_SERVICE; +import static org.apache.iotdb.db.utils.TypeServices.Transformation.IN_TRANSFORMER_SET_INITIALIZER_SERVICE; + public class InTransformer extends UnaryTransformer { private final Satisfy satisfy; @@ -51,49 +54,9 @@ public InTransformer(LayerReader layerReader, boolean isNotIn, Set value } private void initTypedSet(Set values) { - switch (layerReaderDataType) { - case INT32: - case DATE: - intSet = new HashSet<>(); - for (String value : values) { - intSet.add(Integer.valueOf(value)); - } - break; - case INT64: - case TIMESTAMP: - longSet = new HashSet<>(); - for (String value : values) { - longSet.add(Long.valueOf(value)); - } - break; - case FLOAT: - floatSet = new HashSet<>(); - for (String value : values) { - floatSet.add(Float.valueOf(value)); - } - break; - case DOUBLE: - doubleSet = new HashSet<>(); - for (String value : values) { - doubleSet.add(Double.valueOf(value)); - } - break; - case BOOLEAN: - booleanSet = new HashSet<>(); - for (String value : values) { - booleanSet.add(Boolean.valueOf(value)); - } - break; - case TEXT: - case STRING: - stringSet = values; - break; - case BLOB: - case OBJECT: - default: - throw new UnsupportedOperationException( - DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_3 + layerReaderDataType); - } + IN_TRANSFORMER_SET_INITIALIZER_SERVICE + .call(Type.fromTsDataType(layerReaderDataType)) + .initialize(this, values); } @Override @@ -104,37 +67,51 @@ public TSDataType[] getDataTypes() { @Override protected void transform(Column[] columns, ColumnBuilder builder) throws QueryProcessException, IOException { - switch (layerReaderDataType) { - case INT32: - case DATE: - transformInt(columns, builder); - return; - case INT64: - case TIMESTAMP: - transformLong(columns, builder); - return; - case FLOAT: - transformFloat(columns, builder); - return; - case DOUBLE: - transformDouble(columns, builder); - return; - case BOOLEAN: - transformBoolean(columns, builder); - return; - case TEXT: - case STRING: - transformBinary(columns, builder); - return; - case BLOB: - case OBJECT: - default: - throw new QueryProcessException( - DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_3 + layerReaderDataType); + IN_TRANSFORMER_COLUMN_TRANSFORMER_SERVICE + .call(Type.fromTsDataType(layerReaderDataType)) + .transform(this, columns, builder); + } + + public void initIntSet(final Set values) { + intSet = new HashSet<>(); + for (final String value : values) { + intSet.add(Integer.valueOf(value)); + } + } + + public void initLongSet(final Set values) { + longSet = new HashSet<>(); + for (final String value : values) { + longSet.add(Long.valueOf(value)); + } + } + + public void initFloatSet(final Set values) { + floatSet = new HashSet<>(); + for (final String value : values) { + floatSet.add(Float.valueOf(value)); } } - private void transformInt(Column[] columns, ColumnBuilder builder) { + public void initDoubleSet(final Set values) { + doubleSet = new HashSet<>(); + for (final String value : values) { + doubleSet.add(Double.valueOf(value)); + } + } + + public void initBooleanSet(final Set values) { + booleanSet = new HashSet<>(); + for (final String value : values) { + booleanSet.add(Boolean.valueOf(value)); + } + } + + public void initStringSet(final Set values) { + stringSet = values; + } + + public void transformInt(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); int[] values = columns[0].getInts(); boolean[] isNulls = columns[0].isNull(); @@ -149,7 +126,7 @@ private void transformInt(Column[] columns, ColumnBuilder builder) { } } - private void transformLong(Column[] columns, ColumnBuilder builder) { + public void transformLong(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); long[] values = columns[0].getLongs(); boolean[] isNulls = columns[0].isNull(); @@ -164,7 +141,7 @@ private void transformLong(Column[] columns, ColumnBuilder builder) { } } - private void transformFloat(Column[] columns, ColumnBuilder builder) { + public void transformFloat(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); float[] values = columns[0].getFloats(); boolean[] isNulls = columns[0].isNull(); @@ -179,7 +156,7 @@ private void transformFloat(Column[] columns, ColumnBuilder builder) { } } - private void transformDouble(Column[] columns, ColumnBuilder builder) { + public void transformDouble(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); double[] values = columns[0].getDoubles(); boolean[] isNulls = columns[0].isNull(); @@ -194,7 +171,7 @@ private void transformDouble(Column[] columns, ColumnBuilder builder) { } } - private void transformBoolean(Column[] columns, ColumnBuilder builder) { + public void transformBoolean(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); boolean[] values = columns[0].getBooleans(); boolean[] isNulls = columns[0].isNull(); @@ -209,7 +186,7 @@ private void transformBoolean(Column[] columns, ColumnBuilder builder) { } } - private void transformBinary(Column[] columns, ColumnBuilder builder) { + public void transformBinary(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); Binary[] values = columns[0].getBinaries(); boolean[] isNulls = columns[0].isNull(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java index bd8e5b3ca5b0..a570fd5afe2e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java @@ -20,26 +20,19 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.scalar; import org.apache.iotdb.calc.exception.QueryProcessException; -import org.apache.iotdb.calc.transformation.dag.util.CastFunctionUtils; -import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.UnaryTransformer; +import org.apache.iotdb.db.utils.TypeServices.Transformation.CastColumnStrategy; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; -import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; -import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; -import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; -import org.apache.tsfile.read.common.block.column.IntColumnBuilder; -import org.apache.tsfile.read.common.block.column.LongColumnBuilder; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.read.common.type.Type; import java.io.IOException; +import static org.apache.iotdb.db.utils.TypeServices.Transformation.CAST_COLUMN_SERVICE; + public class CastFunctionTransformer extends UnaryTransformer { private final TSDataType targetDataType; @@ -55,502 +48,39 @@ public TSDataType[] getDataTypes() { @Override protected Column[] transform(Column[] columns) throws QueryProcessException, IOException { - switch (layerReaderDataType) { - case INT32: - return castInts(columns); - case INT64: - return castLongs(columns); - case FLOAT: - return castFloats(columns); - case DOUBLE: - return castDoubles(columns); - case BOOLEAN: - return castBooleans(columns); - case TEXT: - return castBinaries(columns); - case BLOB: - case OBJECT: - case STRING: - case TIMESTAMP: - case DATE: - default: - throw new UnsupportedOperationException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_SOURCE_DATATYPE_S_EA03E121, - layerReaderDataType)); - } - } - - private Column[] castInts(Column[] columns) { - if (targetDataType == TSDataType.INT32) { - return columns; - } - - int count = columns[0].getPositionCount(); - int[] values = columns[0].getInts(); - boolean[] isNulls = columns[0].isNull(); - ColumnBuilder builder; - switch (targetDataType) { - case INT64: - builder = new LongColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeLong(values[i]); - } else { - builder.appendNull(); - } - } - break; - case FLOAT: - builder = new FloatColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeFloat(values[i]); - } else { - builder.appendNull(); - } - } - break; - case DOUBLE: - builder = new DoubleColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeDouble(values[i]); - } else { - builder.appendNull(); - } - } - break; - case BOOLEAN: - builder = new BooleanColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeBoolean(values[i] != 0); - } else { - builder.appendNull(); - } - } - break; - case TEXT: - builder = new BinaryColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeBinary(BytesUtils.valueOf(String.valueOf(values[i]))); - } else { - builder.appendNull(); - } - } - break; - case STRING: - case BLOB: - case OBJECT: - case TIMESTAMP: - case DATE: - case INT32: - default: - throw new UnsupportedOperationException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_TARGET_DATATYPE_S_8DEFDAE6, - layerReaderDataType)); - } - - Column valueColumn = builder.build(); - Column timeColumn = columns[1]; - return new Column[] {valueColumn, timeColumn}; - } - - private Column[] castLongs(Column[] columns) { - if (targetDataType == TSDataType.INT64) { - return columns; - } - - int count = columns[0].getPositionCount(); - long[] values = columns[0].getLongs(); - boolean[] isNulls = columns[0].isNull(); - ColumnBuilder builder; - switch (targetDataType) { - case INT32: - builder = new IntColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeInt(CastFunctionUtils.castLongToInt(values[i])); - } else { - builder.appendNull(); - } - } - break; - case FLOAT: - builder = new FloatColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeFloat(values[i]); - } else { - builder.appendNull(); - } - } - break; - case DOUBLE: - builder = new DoubleColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeDouble(values[i]); - } else { - builder.appendNull(); - } - } - break; - case BOOLEAN: - builder = new BooleanColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeBoolean(values[i] != 0L); - } else { - builder.appendNull(); - } - } - break; - case TEXT: - builder = new BinaryColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeBinary(BytesUtils.valueOf(String.valueOf(values[i]))); - } else { - builder.appendNull(); - } - } - break; - case BLOB: - case OBJECT: - case STRING: - case DATE: - case TIMESTAMP: - case INT64: - default: - throw new UnsupportedOperationException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_TARGET_DATATYPE_S_8DEFDAE6, - layerReaderDataType)); - } - - Column valueColumn = builder.build(); - Column timeColumn = columns[1]; - return new Column[] {valueColumn, timeColumn}; - } - - private Column[] castFloats(Column[] columns) { - if (targetDataType == TSDataType.FLOAT) { - return columns; - } - - int count = columns[0].getPositionCount(); - float[] values = columns[0].getFloats(); - boolean[] isNulls = columns[0].isNull(); - ColumnBuilder builder; - switch (targetDataType) { - case INT32: - builder = new IntColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeInt(CastFunctionUtils.castFloatToInt(values[i])); - } else { - builder.appendNull(); - } - } - break; - case INT64: - builder = new LongColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeLong(CastFunctionUtils.castFloatToLong(values[i])); - } else { - builder.appendNull(); - } - } - break; - case DOUBLE: - builder = new DoubleColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeDouble(values[i]); - } else { - builder.appendNull(); - } - } - break; - case BOOLEAN: - builder = new BooleanColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeBoolean(values[i] != 0f); - } else { - builder.appendNull(); - } - } - break; - case TEXT: - builder = new BinaryColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeBinary(BytesUtils.valueOf(String.valueOf(values[i]))); - } else { - builder.appendNull(); - } - } - break; - case BLOB: - case OBJECT: - case STRING: - case TIMESTAMP: - case DATE: - case FLOAT: - default: - throw new UnsupportedOperationException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_TARGET_DATATYPE_S_8DEFDAE6, - layerReaderDataType)); - } - - Column valueColumn = builder.build(); - Column timeColumn = columns[1]; - return new Column[] {valueColumn, timeColumn}; - } - - private Column[] castDoubles(Column[] columns) { - if (targetDataType == TSDataType.DOUBLE) { - return columns; - } - - int count = columns[0].getPositionCount(); - double[] values = columns[0].getDoubles(); - boolean[] isNulls = columns[0].isNull(); - ColumnBuilder builder; - switch (targetDataType) { - case INT32: - builder = new IntColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeInt(CastFunctionUtils.castDoubleToInt(values[i])); - } else { - builder.appendNull(); - } - } - break; - case INT64: - builder = new LongColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeLong(CastFunctionUtils.castDoubleToLong(values[i])); - } else { - builder.appendNull(); - } - } - break; - case FLOAT: - builder = new FloatColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeFloat(CastFunctionUtils.castDoubleToFloat(values[i])); - } else { - builder.appendNull(); - } - } - break; - case BOOLEAN: - builder = new BooleanColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeBoolean(values[i] != 0.0); - } else { - builder.appendNull(); - } - } - break; - case TEXT: - builder = new BinaryColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeBinary(BytesUtils.valueOf(String.valueOf(values[i]))); - } else { - builder.appendNull(); - } - } - break; - case BLOB: - case OBJECT: - case STRING: - case TIMESTAMP: - case DATE: - case DOUBLE: - default: - throw new UnsupportedOperationException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_TARGET_DATATYPE_S_8DEFDAE6, - layerReaderDataType)); + final Type sourceType; + try { + sourceType = Type.fromTsDataType(layerReaderDataType); + } catch (final UnsupportedOperationException ignored) { + throw new UnsupportedOperationException( + String.format("Unsupported source dataType: %s", layerReaderDataType)); } - Column valueColumn = builder.build(); - Column timeColumn = columns[1]; - return new Column[] {valueColumn, timeColumn}; - } - - private Column[] castBooleans(Column[] columns) { - if (targetDataType == TSDataType.BOOLEAN) { - return columns; + final Type targetType; + try { + targetType = Type.fromTsDataType(targetDataType); + } catch (final UnsupportedOperationException ignored) { + throw new UnsupportedOperationException( + String.format("Unsupported target dataType: %s", layerReaderDataType)); } - int count = columns[0].getPositionCount(); - boolean[] values = columns[0].getBooleans(); - boolean[] isNulls = columns[0].isNull(); - ColumnBuilder builder; - switch (targetDataType) { - case INT32: - builder = new IntColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeInt(values[i] ? 1 : 0); - } else { - builder.appendNull(); - } - } - break; - case INT64: - builder = new LongColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeLong(values[i] ? 1L : 0); - } else { - builder.appendNull(); - } - } - break; - case FLOAT: - builder = new FloatColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeFloat(values[i] ? 1.0f : 0); - } else { - builder.appendNull(); - } - } - break; - case DOUBLE: - builder = new DoubleColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeDouble(values[i] ? 1.0 : 0); - } else { - builder.appendNull(); - } - } - break; - case TEXT: - builder = new BinaryColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - builder.writeBinary(BytesUtils.valueOf(String.valueOf(values[i]))); - } else { - builder.appendNull(); - } - } - break; - case STRING: - case BLOB: - case OBJECT: - case DATE: - case TIMESTAMP: - case BOOLEAN: - default: - throw new UnsupportedOperationException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_TARGET_DATATYPE_S_8DEFDAE6, - layerReaderDataType)); - } - - Column valueColumn = builder.build(); - Column timeColumn = columns[1]; - return new Column[] {valueColumn, timeColumn}; - } - - private Column[] castBinaries(Column[] columns) { - if (targetDataType == TSDataType.TEXT) { + final CastColumnStrategy strategy = CAST_COLUMN_SERVICE.call(sourceType).call(targetType); + strategy.validate(); + if (layerReaderDataType == targetDataType) { return columns; } - int count = columns[0].getPositionCount(); - Binary[] values = columns[0].getBinaries(); - boolean[] isNulls = columns[0].isNull(); - ColumnBuilder builder; - switch (targetDataType) { - case INT32: - builder = new IntColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - String str = values[i].getStringValue(TSFileConfig.STRING_CHARSET); - builder.writeInt(Integer.parseInt(str)); - } else { - builder.appendNull(); - } - } - break; - case INT64: - builder = new LongColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - String str = values[i].getStringValue(TSFileConfig.STRING_CHARSET); - builder.writeLong(Long.parseLong(str)); - } else { - builder.appendNull(); - } - } - break; - case FLOAT: - builder = new FloatColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - String str = values[i].getStringValue(TSFileConfig.STRING_CHARSET); - builder.writeFloat(CastFunctionUtils.castTextToFloat(str)); - } else { - builder.appendNull(); - } - } - break; - case DOUBLE: - builder = new DoubleColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - String str = values[i].getStringValue(TSFileConfig.STRING_CHARSET); - builder.writeDouble(CastFunctionUtils.castTextToDouble(str)); - } else { - builder.appendNull(); - } - } - break; - case BOOLEAN: - builder = new BooleanColumnBuilder(null, count); - for (int i = 0; i < count; i++) { - if (!isNulls[i]) { - String str = values[i].getStringValue(TSFileConfig.STRING_CHARSET); - builder.writeBoolean(CastFunctionUtils.castTextToBoolean(str)); - } else { - builder.appendNull(); - } - } - break; - case TIMESTAMP: - case DATE: - case STRING: - case BLOB: - case OBJECT: - case TEXT: - default: - throw new UnsupportedOperationException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_TARGET_DATATYPE_S_8DEFDAE6, - layerReaderDataType)); + final Column valueColumn = columns[0]; + final int positionCount = valueColumn.getPositionCount(); + final boolean[] isNulls = valueColumn.isNull(); + final ColumnBuilder builder = strategy.createBuilder(positionCount); + for (int i = 0; i < positionCount; i++) { + if (isNulls[i]) { + builder.appendNull(); + } else { + strategy.cast(valueColumn, i, builder); + } } - - Column valueColumn = builder.build(); - Column timeColumn = columns[1]; - return new Column[] {valueColumn, timeColumn}; + return new Column[] {builder.build(), columns[1]}; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java index 4bce7c9ec2f5..21244baa1617 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java @@ -27,8 +27,12 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; + +import static org.apache.iotdb.db.utils.TypeServices.Transformation.DIFF_TRANSFORMER_SERVICE; public class DiffFunctionTransformer extends UnaryTransformer { + private final boolean ignoreNull; // cache the last non-null value @@ -49,33 +53,17 @@ public TSDataType[] getDataTypes() { @Override public void transform(Column[] columns, ColumnBuilder builder) throws QueryProcessException { - switch (layerReaderDataType) { - case INT32: - transformInt(columns, builder); - return; - case INT64: - transformLong(columns, builder); - return; - case FLOAT: - transformFloat(columns, builder); - return; - case DOUBLE: - transformDouble(columns, builder); - return; - case BLOB: - case OBJECT: - case TEXT: - case DATE: - case STRING: - case TIMESTAMP: - case BOOLEAN: - default: - throw new QueryProcessException( - DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + layerReaderDataType); + final Type type; + try { + type = Type.fromTsDataType(layerReaderDataType); + } catch (UnsupportedOperationException ignored) { + throw new QueryProcessException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + layerReaderDataType); } + DIFF_TRANSFORMER_SERVICE.call(type).transform(this, columns, builder); } - private void transformInt(Column[] columns, ColumnBuilder builder) { + public void transformInt(Column[] columns, ColumnBuilder builder) { // Position count at first iteration is count // Then it become count - 1 at latter iteration int count = columns[0].getPositionCount(); @@ -99,7 +87,7 @@ private void transformInt(Column[] columns, ColumnBuilder builder) { } } - private void transformLong(Column[] columns, ColumnBuilder builder) { + public void transformLong(Column[] columns, ColumnBuilder builder) { // Position count at first iteration is count // Then it become count - 1 at latter iteration int count = columns[0].getPositionCount(); @@ -123,7 +111,7 @@ private void transformLong(Column[] columns, ColumnBuilder builder) { } } - private void transformFloat(Column[] columns, ColumnBuilder builder) { + public void transformFloat(Column[] columns, ColumnBuilder builder) { // Position count at first iteration is count // Then it become count - 1 at latter iteration int count = columns[0].getPositionCount(); @@ -147,7 +135,7 @@ private void transformFloat(Column[] columns, ColumnBuilder builder) { } } - private void transformDouble(Column[] columns, ColumnBuilder builder) { + public void transformDouble(Column[] columns, ColumnBuilder builder) { // Position count at first iteration is count // Then it become count - 1 at latter iteration int count = columns[0].getPositionCount(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java index 4d2a41b85bcd..f1f79b1c1008 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java @@ -20,17 +20,20 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.scalar; import org.apache.iotdb.calc.exception.QueryProcessException; -import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.UnaryTransformer; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import java.io.IOException; +import static org.apache.iotdb.db.utils.TypeServices.Transformation.ROUND_TRANSFORMER_SERVICE; + public class RoundFunctionTransformer extends UnaryTransformer { + private final TSDataType targetDataType; protected int places; @@ -49,35 +52,17 @@ public TSDataType[] getDataTypes() { @Override protected void transform(Column[] columns, ColumnBuilder builder) throws QueryProcessException, IOException { - switch (layerReaderDataType) { - case INT32: - transformInt(columns, builder); - return; - case INT64: - transformLong(columns, builder); - return; - case FLOAT: - transformFloat(columns, builder); - return; - case DOUBLE: - transformDouble(columns, builder); - return; - case TIMESTAMP: - case BOOLEAN: - case DATE: - case STRING: - case TEXT: - case BLOB: - case OBJECT: - default: - throw new UnsupportedOperationException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_UNSUPPORTED_SOURCE_DATATYPE_S_EA03E121, - layerReaderDataType)); + final Type type; + try { + type = Type.fromTsDataType(layerReaderDataType); + } catch (UnsupportedOperationException ignored) { + throw new UnsupportedOperationException( + String.format("Unsupported source dataType: %s", layerReaderDataType)); } + ROUND_TRANSFORMER_SERVICE.call(type).transform(this, columns, builder); } - private void transformInt(Column[] columns, ColumnBuilder builder) { + public void transformInt(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); int[] values = columns[0].getInts(); @@ -92,7 +77,7 @@ private void transformInt(Column[] columns, ColumnBuilder builder) { } } - private void transformLong(Column[] columns, ColumnBuilder builder) { + public void transformLong(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); long[] values = columns[0].getLongs(); @@ -107,7 +92,7 @@ private void transformLong(Column[] columns, ColumnBuilder builder) { } } - private void transformFloat(Column[] columns, ColumnBuilder builder) { + public void transformFloat(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); float[] values = columns[0].getFloats(); @@ -122,7 +107,7 @@ private void transformFloat(Column[] columns, ColumnBuilder builder) { } } - private void transformDouble(Column[] columns, ColumnBuilder builder) { + public void transformDouble(Column[] columns, ColumnBuilder builder) { int count = columns[0].getPositionCount(); double[] values = columns[0].getDoubles(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java index 0755b1be52b7..3ceb910e9595 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java @@ -24,20 +24,13 @@ import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand; import org.apache.iotdb.db.queryengine.transformation.datastructure.util.ValueRecorder; import org.apache.iotdb.db.utils.CommonUtils; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.block.column.BinaryColumn; -import org.apache.tsfile.read.common.block.column.BooleanColumn; -import org.apache.tsfile.read.common.block.column.DoubleColumn; -import org.apache.tsfile.read.common.block.column.FloatColumn; -import org.apache.tsfile.read.common.block.column.IntColumn; -import org.apache.tsfile.read.common.block.column.LongColumn; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.read.common.type.Type; import java.util.Objects; -import java.util.Optional; public class TransformUtils { @@ -54,33 +47,13 @@ public static Column transformConstantOperandToColumn(ConstantOperand constantOp CommonUtils.parseValue(constantOperand.getDataType(), constantOperand.getValueString()); if (value == null) { throw new UnsupportedOperationException( - String.format( - DataNodeQueryMessages.QUERY_EXCEPTION_INVALID_CONSTANT_OPERAND_S_939F3B8D, - constantOperand.getExpressionString())); + DataNodeQueryMessages.UNSUPPORTED_CONSTANT_OPERAND + + constantOperand.getExpressionString()); } - switch (constantOperand.getDataType()) { - case INT32: - return new IntColumn(1, Optional.empty(), new int[] {(int) value}); - case INT64: - return new LongColumn(1, Optional.empty(), new long[] {(long) value}); - case FLOAT: - return new FloatColumn(1, Optional.empty(), new float[] {(float) value}); - case DOUBLE: - return new DoubleColumn(1, Optional.empty(), new double[] {(double) value}); - case TEXT: - return new BinaryColumn(1, Optional.empty(), new Binary[] {(Binary) value}); - case BOOLEAN: - return new BooleanColumn(1, Optional.empty(), new boolean[] {(boolean) value}); - case STRING: - case BLOB: - case OBJECT: - case DATE: - case TIMESTAMP: - default: - throw new UnSupportedDataTypeException( - DataNodeQueryMessages.UNSUPPORTED_TYPE + constantOperand.getDataType()); - } + return TypeServices.Transformation.CONSTANT_COLUMN_BUILDER_SERVICE + .call(Type.fromTsDataType(constantOperand.getDataType())) + .apply(value); } catch (QueryProcessException e) { throw new UnsupportedOperationException(e); } @@ -88,80 +61,8 @@ public static Column transformConstantOperandToColumn(ConstantOperand constantOp public static boolean splitWindowForStateWindow( TSDataType dataType, ValueRecorder valueRecorder, double delta, Column values, int index) { - boolean res; - switch (dataType) { - case INT32: - if (!valueRecorder.hasRecorded()) { - valueRecorder.recordInt(values.getInt(index - 1)); - valueRecorder.setRecorded(true); - } - res = Math.abs(values.getInt(index) - valueRecorder.getInt()) > delta; - if (res) { - valueRecorder.recordInt(values.getInt(index)); - } - break; - case INT64: - if (!valueRecorder.hasRecorded()) { - valueRecorder.recordLong(values.getLong(index - 1)); - valueRecorder.setRecorded(true); - } - res = Math.abs(values.getLong(index) - valueRecorder.getLong()) > delta; - if (res) { - valueRecorder.recordLong(values.getLong(index)); - } - break; - case FLOAT: - if (!valueRecorder.hasRecorded()) { - valueRecorder.recordFloat(values.getFloat(index - 1)); - valueRecorder.setRecorded(true); - } - res = Math.abs(values.getFloat(index) - valueRecorder.getFloat()) > delta; - if (res) { - valueRecorder.recordFloat(values.getFloat(index)); - } - break; - case DOUBLE: - if (!valueRecorder.hasRecorded()) { - valueRecorder.recordDouble(values.getDouble(index - 1)); - valueRecorder.setRecorded(true); - } - res = Math.abs(values.getDouble(index) - valueRecorder.getDouble()) > delta; - if (res) { - valueRecorder.recordDouble(values.getDouble(index)); - } - break; - case BOOLEAN: - if (!valueRecorder.hasRecorded()) { - valueRecorder.recordBoolean(values.getBoolean(index - 1)); - valueRecorder.setRecorded(true); - } - res = values.getBoolean(index) != valueRecorder.getBoolean(); - if (res) { - valueRecorder.recordBoolean(values.getBoolean(index)); - } - break; - case TEXT: - if (!valueRecorder.hasRecorded()) { - Binary binary = values.getBinary(index - 1); - valueRecorder.recordString(binary.toString()); - valueRecorder.setRecorded(true); - } - String str = values.getBinary(index).toString(); - res = !str.equals(valueRecorder.getString()); - if (res) { - valueRecorder.recordString(str); - } - break; - case TIMESTAMP: - case DATE: - case BLOB: - case OBJECT: - case STRING: - default: - throw new UnsupportedOperationException( - DataNodeQueryMessages - .QUERY_EXCEPTION_THE_DATA_TYPE_OF_THE_STATE_WINDOW_STRATEGY_IS_NOT_VALID_DFFBF210); - } - return res; + return TypeServices.Transformation.STATE_WINDOW_SPLITTER_SERVICE + .call(Type.fromTsDataType(dataType)) + .split(valueRecorder, delta, values, index); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java index e59e088e1dd3..3593e96528b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java @@ -20,69 +20,24 @@ package org.apache.iotdb.db.queryengine.transformation.dag.util; import org.apache.iotdb.calc.exception.QueryProcessException; -import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; -import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; -import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; -import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; -import org.apache.tsfile.read.common.block.column.IntColumnBuilder; -import org.apache.tsfile.read.common.block.column.LongColumnBuilder; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.read.common.type.Type; public class TypeUtils { public static ColumnBuilder initColumnBuilder(TSDataType type, int count) { - switch (type) { - case INT32: - return new IntColumnBuilder(null, count, TSDataType.INT32); - case DATE: - return new IntColumnBuilder(null, count, TSDataType.DATE); - case INT64: - case TIMESTAMP: - return new LongColumnBuilder(null, count); - case FLOAT: - return new FloatColumnBuilder(null, count); - case DOUBLE: - return new DoubleColumnBuilder(null, count); - case BOOLEAN: - return new BooleanColumnBuilder(null, count); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return new BinaryColumnBuilder(null, count); - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_DO_NOT_SUPPORT_CREATE_COLUMNBUILDER_WITH_DATA_TYPE_S_1672578A, - type)); - } + return TypeServices.Transformation.COLUMN_BUILDER_SERVICE + .call(Type.fromTsDataType(type)) + .apply(count); } public static double castValueToDouble(Column column, TSDataType type, int index) throws QueryProcessException { - switch (type) { - case INT32: - case DATE: - return column.getInt(index); - case INT64: - case TIMESTAMP: - return column.getLong(index); - case FLOAT: - return column.getFloat(index); - case DOUBLE: - return column.getDouble(index); - case BOOLEAN: - return column.getBoolean(index) ? 1 : 0; - case BLOB: - case STRING: - case TEXT: - default: - throw new QueryProcessException(DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type); - } + return TypeServices.Transformation.VALUE_TO_DOUBLE_SERVICE + .call(Type.fromTsDataType(type)) + .convert(column, index); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java index 5e72bc481e9e..947a22ae6002 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.utils.MetadataUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionLastTimeCheckFailedException; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary; @@ -56,6 +55,7 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.read.reader.page.TimePageReader; import org.apache.tsfile.read.reader.page.ValuePageReader; @@ -663,38 +663,14 @@ private boolean canFlushPage(PageLoader timePage, List valuePages) { private long estimateMemorySizeAsPageWriter(PageLoader pageLoader) { long count = pageLoader.getHeader().getStatistics().getCount(); - long size; - switch (pageLoader.getDataType()) { - case INT32: - case DATE: - size = count * Integer.BYTES; - break; - case TIMESTAMP: - case INT64: - case VECTOR: - size = count * Long.BYTES; - break; - case FLOAT: - size = count * Float.BYTES; - break; - case DOUBLE: - size = count * Double.BYTES; - break; - case BOOLEAN: - size = count * Byte.BYTES; - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - size = pageLoader.getHeader().getUncompressedSize(); - break; - default: - throw new IllegalArgumentException( - String.format( - StorageEngineMessages.STORAGE_EXCEPTION_UNSUPPORTED_DATA_TYPE_S_D16A1E9A, - pageLoader.getDataType().toString())); - } + TSDataType dataType = pageLoader.getDataType(); + long size = + dataType.isBinary() + ? pageLoader.getHeader().getUncompressedSize() + : count + * (dataType == TSDataType.VECTOR + ? Long.BYTES + : Type.fromTsDataType(dataType).estimateValueSize()); // Due to the fact that the page writer in memory includes some other objects // and has a special calculation method, the estimated size will actually be // larger. So we simply adopt the method of multiplying by 1.05 times. If this diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java index c5115e46be50..44f8427a6f90 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java @@ -21,7 +21,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; -import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionLastTimeCheckFailedException; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary; import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileWriter; @@ -36,6 +35,7 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.IChunkReader; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.read.reader.chunk.ChunkReader; @@ -337,34 +337,8 @@ private void mergeWithCachedChunk(Chunk currentChunk, ChunkMetadata currentChunk } private void writeTimeAndValueToChunkWriter(TimeValuePair timeValuePair) { - switch (chunkWriter.getDataType()) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary()); - break; - case FLOAT: - chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getFloat()); - break; - case DOUBLE: - chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getDouble()); - break; - case BOOLEAN: - chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBoolean()); - break; - case INT64: - case TIMESTAMP: - chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getLong()); - break; - case INT32: - case DATE: - chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getInt()); - break; - default: - throw new UnsupportedOperationException( - StorageEngineMessages.UNKNOWN_DATA_TYPE + chunkWriter.getDataType()); - } + Type.fromTsDataType(chunkWriter.getDataType()) + .write(chunkWriter, timeValuePair.getTimestamp(), timeValuePair.getValue()); } private void flushChunkToFileWriter(Chunk chunk, ChunkMetadata chunkMetadata) throws IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java index 56ee94acf0c7..da85dcbbce44 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionLastTimeCheckFailedException; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.batch.utils.FollowingBatchCompactionAlignedChunkWriter; @@ -43,6 +42,7 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; import org.apache.tsfile.write.chunk.ChunkWriterImpl; @@ -177,49 +177,18 @@ private boolean containsVariableLengthType(IChunkWriter chunkWriter) { protected void writeDataPoint( long timestamp, TsPrimitiveType value, IChunkWriter chunkWriter, int subTaskId) { - long writtenPointTotalSize = 0; + long writtenPointTotalSize; if (chunkWriter instanceof ChunkWriterImpl) { ChunkWriterImpl chunkWriterImpl = (ChunkWriterImpl) chunkWriter; - switch (chunkWriterImpl.getDataType()) { - case TEXT: - case STRING: - case BLOB: - case OBJECT: - chunkWriterImpl.write(timestamp, value.getBinary()); - writtenPointTotalSize += value.getBinary().getLength(); - break; - case DOUBLE: - chunkWriterImpl.write(timestamp, value.getDouble()); - break; - case BOOLEAN: - chunkWriterImpl.write(timestamp, value.getBoolean()); - break; - case INT64: - case TIMESTAMP: - chunkWriterImpl.write(timestamp, value.getLong()); - break; - case INT32: - case DATE: - chunkWriterImpl.write(timestamp, value.getInt()); - break; - case FLOAT: - chunkWriterImpl.write(timestamp, value.getFloat()); - break; - default: - throw new UnsupportedOperationException( - StorageEngineMessages.UNKNOWN_DATA_TYPE + chunkWriterImpl.getDataType()); - } + Type.fromTsDataType(chunkWriterImpl.getDataType()).write(chunkWriterImpl, timestamp, value); + writtenPointTotalSize = Long.BYTES + estimateWrittenValueSize(value); } else { AlignedChunkWriterImpl alignedChunkWriter = (AlignedChunkWriterImpl) chunkWriter; alignedChunkWriter.write(timestamp, value.getVector()); - if (hasVariableLengthTypeArray[subTaskId]) { - writtenPointTotalSize = estimateWrittenPointTotalSize(value); - } + writtenPointTotalSize = estimateWrittenPointTotalSize(value); } chunkPointNumArray[subTaskId]++; - if (hasVariableLengthTypeArray[subTaskId]) { - writtenPointTotalSizeArray[subTaskId] += writtenPointTotalSize; - } + writtenPointTotalSizeArray[subTaskId] += writtenPointTotalSize; } private long estimateWrittenPointTotalSize(TsPrimitiveType value) { @@ -229,30 +198,7 @@ private long estimateWrittenPointTotalSize(TsPrimitiveType value) { if (tsPrimitiveType == null) { continue; } - TSDataType dataType = tsPrimitiveType.getDataType(); - switch (dataType) { - case TEXT: - case STRING: - case BLOB: - case OBJECT: - size += tsPrimitiveType.getBinary().getLength(); - break; - case DOUBLE: - case INT64: - case TIMESTAMP: - size += Long.BYTES; - break; - case INT32: - case DATE: - case FLOAT: - size += Integer.BYTES; - break; - case BOOLEAN: - size += 1; - break; - default: - break; - } + size += estimateWrittenValueSize(tsPrimitiveType); } return size; } @@ -263,37 +209,29 @@ protected long estimateWrittenPointTotalSize(TsBlock tsBlock) { Column[] columns = tsBlock.getValueColumns(); for (Column column : columns) { TSDataType dataType = column.getDataType(); + Type type = Type.fromTsDataType(dataType); if (dataType.isBinary()) { for (int j = 0; j < pointCount; j++) { if (column.isNull(j)) { continue; } - size += column.getBinary(j).getLength(); + // The checkpoint estimate historically excludes the binary length prefix. + size += type.calcTypeSize(type.getObject(column, j)) - Integer.BYTES; } continue; } // This is only used as a checkpoint estimate, so fixed-width values use count directly. - switch (dataType) { - case DOUBLE: - case INT64: - case TIMESTAMP: - size += (long) Long.BYTES * pointCount; - break; - case INT32: - case DATE: - case FLOAT: - size += (long) Integer.BYTES * pointCount; - break; - case BOOLEAN: - size += pointCount; - break; - default: - break; - } + size += type.estimateValueSize() * pointCount; } return size; } + private long estimateWrittenValueSize(TsPrimitiveType value) { + int valueSize = Type.fromTsDataType(value.getDataType()).calcTypeSize(value); + // The checkpoint estimate historically excludes the binary length prefix. + return value.getDataType().isBinary() ? valueSize - Integer.BYTES : valueSize; + } + @SuppressWarnings("squid:S2445") protected void sealChunk( CompactionTsFileWriter targetWriter, IChunkWriter chunkWriter, int subTaskId) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index 70b5a5fb8d18..f15831baf6e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.memtable; -import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; @@ -29,7 +28,6 @@ import org.apache.iotdb.db.utils.datastructure.MemPointIteratorFactory; import org.apache.iotdb.db.utils.datastructure.TVList; -import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; @@ -42,10 +40,10 @@ import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.schema.IMeasurementSchema; import java.io.IOException; @@ -155,7 +153,9 @@ public void initChunkMetaFromTvLists(Filter globalTimeFilter) { IChunkMetadata timeChunkMetadata = new ChunkMetadata(timeChunkName, TSDataType.VECTOR, null, null, 0, chunkTimeStatistics); Statistics[] chunkValueStatistics = new Statistics[dataTypes.size()]; + Type[] valueTypes = new Type[dataTypes.size()]; for (int column = 0; column < dataTypes.size(); column++) { + valueTypes[column] = Type.fromTsDataType(dataTypes.get(column)); chunkValueStatistics[column] = Statistics.getStatsByType(dataTypes.get(column)); } @@ -183,84 +183,14 @@ public void initChunkMetaFromTvLists(Filter globalTimeFilter) { } // value columns for (int column = 0; column < tsBlock.getValueColumnCount(); column++) { - Statistics pageValueStats = - Statistics.getStatsByType(dataTypes.get(column)); - switch (dataTypes.get(column)) { - case BOOLEAN: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - if (tsBlock.getColumn(column).isNull(i)) { - continue; - } - pageValueStats.update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getBoolean(i)); - chunkValueStatistics[column].update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getBoolean(i)); - } - break; - case INT32: - case DATE: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - if (tsBlock.getColumn(column).isNull(i)) { - continue; - } - pageValueStats.update(tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getInt(i)); - chunkValueStatistics[column].update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getInt(i)); - } - break; - case INT64: - case TIMESTAMP: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - if (tsBlock.getColumn(column).isNull(i)) { - continue; - } - pageValueStats.update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getLong(i)); - chunkValueStatistics[column].update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getLong(i)); - } - break; - case FLOAT: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - if (tsBlock.getColumn(column).isNull(i)) { - continue; - } - pageValueStats.update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getFloat(i)); - chunkValueStatistics[column].update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getFloat(i)); - } - break; - case DOUBLE: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - if (tsBlock.getColumn(column).isNull(i)) { - continue; - } - pageValueStats.update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getDouble(i)); - chunkValueStatistics[column].update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getDouble(i)); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - if (tsBlock.getColumn(column).isNull(i)) { - continue; - } - pageValueStats.update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getBinary(i)); - chunkValueStatistics[column].update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getBinary(i)); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - StorageEngineMessages.STORAGE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataTypes.get(column))); + Statistics pageValueStats = pageValueStatistics[column]; + Type valueType = valueTypes[column]; + for (int i = 0; i < tsBlock.getPositionCount(); i++) { + if (tsBlock.getColumn(column).isNull(i)) { + continue; + } + valueType.update(pageValueStats, tsBlock, column, i); + valueType.update(chunkValueStatistics[column], tsBlock, column, i); } pageValueStatistics[column] = pageValueStats.isEmpty() ? null : pageValueStats; } @@ -420,6 +350,10 @@ private TsBlock buildTsBlock() { private void writeValidValuesIntoTsBlock(TsBlockBuilder builder) throws IOException { MemPointIterator timeValuePairIterator = createMemPointIterator(Ordering.ASC, null); + Type[] valueTypes = new Type[dataTypes.size()]; + for (int columnIndex = 0; columnIndex < dataTypes.size(); columnIndex++) { + valueTypes[columnIndex] = Type.fromTsDataType(dataTypes.get(columnIndex)); + } while (timeValuePairIterator.hasNextTimeValuePair()) { TimeValuePair tvPair = timeValuePairIterator.nextTimeValuePair(); @@ -433,34 +367,8 @@ private void writeValidValuesIntoTsBlock(TsBlockBuilder builder) throws IOExcept builder.getColumnBuilder(columnIndex).appendNull(); continue; } - ColumnBuilder valueBuilder = builder.getColumnBuilder(columnIndex); - switch (dataTypes.get(columnIndex)) { - case BOOLEAN: - valueBuilder.writeBoolean(values[columnIndex].getBoolean()); - break; - case INT32: - case DATE: - valueBuilder.writeInt(values[columnIndex].getInt()); - break; - case INT64: - case TIMESTAMP: - valueBuilder.writeLong(values[columnIndex].getLong()); - break; - case FLOAT: - valueBuilder.writeFloat(values[columnIndex].getFloat()); - break; - case DOUBLE: - valueBuilder.writeDouble(values[columnIndex].getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueBuilder.writeBinary(values[columnIndex].getBinary()); - break; - default: - break; - } + valueTypes[columnIndex].writeObject( + builder.getColumnBuilder(columnIndex), values[columnIndex].getValue()); } builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index d1c153841733..723cd266c1c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo; import org.apache.iotdb.db.utils.datastructure.MemPointIterator; @@ -37,6 +38,7 @@ import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; @@ -701,6 +703,12 @@ private void handleEncodingWithoutDeletedMeasurements( } List dataTypes = alignedWorkingListForFlush.getTsDataTypes(); + List valueWriters = new ArrayList<>(dataTypes.size()); + for (TSDataType dataType : dataTypes) { + valueWriters.add( + TypeServices.StorageEngine.ALIGNED_TV_LIST_CHUNK_WRITER_SERVICE.call( + Type.fromTsDataType(dataType))); + } Pair[] lastValidPointIndexForTimeDupCheck = new Pair[dataTypes.size()]; for (List pageRange : chunkRange) { AlignedChunkWriterImpl alignedChunkWriter = @@ -712,7 +720,9 @@ private void handleEncodingWithoutDeletedMeasurements( && lastValidPointIndexForTimeDupCheck[columnIndex] == null) { lastValidPointIndexForTimeDupCheck[columnIndex] = new Pair<>(Long.MIN_VALUE, null); } - TSDataType tsDataType = dataTypes.get(columnIndex); + TypeServices.AlignedTVListChunkWriter valueWriter = valueWriters.get(columnIndex); + ValueChunkWriter valueChunkWriter = + alignedChunkWriter.getValueChunkWriterByIndex(columnIndex); for (int sortedRowIndex = pageRange.get(pageNum * 2); sortedRowIndex <= pageRange.get(pageNum * 2 + 1); sortedRowIndex++) { @@ -754,68 +764,13 @@ private void handleEncodingWithoutDeletedMeasurements( } boolean isNull = alignedWorkingListForFlush.isNullValue(originRowIndex, columnIndex); - switch (tsDataType) { - case BOOLEAN: - alignedChunkWriter.writeByColumn( - time, - !isNull - && alignedWorkingListForFlush.getBooleanByValueIndex( - originRowIndex, columnIndex), - isNull); - break; - case INT32: - case DATE: - alignedChunkWriter.writeByColumn( - time, - isNull - ? 0 - : alignedWorkingListForFlush.getIntByValueIndex( - originRowIndex, columnIndex), - isNull); - break; - case INT64: - case TIMESTAMP: - alignedChunkWriter.writeByColumn( - time, - isNull - ? 0 - : alignedWorkingListForFlush.getLongByValueIndex( - originRowIndex, columnIndex), - isNull); - break; - case FLOAT: - alignedChunkWriter.writeByColumn( - time, - isNull - ? 0 - : alignedWorkingListForFlush.getFloatByValueIndex( - originRowIndex, columnIndex), - isNull); - break; - case DOUBLE: - alignedChunkWriter.writeByColumn( - time, - isNull - ? 0 - : alignedWorkingListForFlush.getDoubleByValueIndex( - originRowIndex, columnIndex), - isNull); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - alignedChunkWriter.writeByColumn( - time, - isNull - ? null - : alignedWorkingListForFlush.getBinaryByValueIndex( - originRowIndex, columnIndex), - isNull); - break; - default: - break; - } + valueWriter.write( + valueChunkWriter, + time, + alignedWorkingListForFlush, + originRowIndex, + columnIndex, + isNull); } alignedChunkWriter.nextColumn(); } @@ -948,35 +903,9 @@ private void writeValueFromArray( Object valueArray, int elementIndex, boolean isNull) { - switch (dataType) { - case BOOLEAN -> - valueChunkWriter.write( - time, - !isNull && valueArray != null && ((boolean[]) valueArray)[elementIndex], - isNull); - case INT32, DATE -> - valueChunkWriter.write( - time, isNull || valueArray == null ? 0 : ((int[]) valueArray)[elementIndex], isNull); - case INT64, TIMESTAMP -> - valueChunkWriter.write( - time, isNull || valueArray == null ? 0 : ((long[]) valueArray)[elementIndex], isNull); - case FLOAT -> - valueChunkWriter.write( - time, - isNull || valueArray == null ? 0 : ((float[]) valueArray)[elementIndex], - isNull); - case DOUBLE -> - valueChunkWriter.write( - time, - isNull || valueArray == null ? 0 : ((double[]) valueArray)[elementIndex], - isNull); - case TEXT, STRING, BLOB, OBJECT -> - valueChunkWriter.write( - time, - isNull || valueArray == null ? null : ((Binary[]) valueArray)[elementIndex], - isNull); - case VECTOR, UNKNOWN -> throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + dataType); - } + TypeServices.StorageEngine.VALUE_CHUNK_ARRAY_WRITER_SERVICE + .call(Type.fromTsDataType(dataType)) + .write(valueChunkWriter, time, valueArray, elementIndex, isNull); } private void writeValuesFromArray( @@ -987,27 +916,9 @@ private void writeValuesFromArray( boolean[] nulls, int arrayOffset, int pointsInSegment) { - switch (dataType) { - case BOOLEAN -> - valueChunkWriter.write( - timestamps, (boolean[]) valueArray, nulls, pointsInSegment, arrayOffset); - case INT32, DATE -> - valueChunkWriter.write( - timestamps, (int[]) valueArray, nulls, pointsInSegment, arrayOffset); - case INT64, TIMESTAMP -> - valueChunkWriter.write( - timestamps, (long[]) valueArray, nulls, pointsInSegment, arrayOffset); - case FLOAT -> - valueChunkWriter.write( - timestamps, (float[]) valueArray, nulls, pointsInSegment, arrayOffset); - case DOUBLE -> - valueChunkWriter.write( - timestamps, (double[]) valueArray, nulls, pointsInSegment, arrayOffset); - case TEXT, STRING, BLOB, OBJECT -> - valueChunkWriter.write( - timestamps, (Binary[]) valueArray, nulls, pointsInSegment, arrayOffset); - case VECTOR, UNKNOWN -> throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + dataType); - } + TypeServices.StorageEngine.VALUE_CHUNK_ARRAY_BATCH_WRITER_SERVICE + .call(Type.fromTsDataType(dataType)) + .write(valueChunkWriter, timestamps, valueArray, nulls, arrayOffset, pointsInSegment); } private void writeValuesFromArray( @@ -1018,71 +929,9 @@ private void writeValuesFromArray( BitMap bitMap, int arrayOffset, int pointsInSegment) { - // Keep null detection in the bitmap so the page writer can skip encoding and statistics - // updates without materializing a temporary boolean array. - switch (dataType) { - case BOOLEAN -> - valueChunkWriter - .getPageWriter() - .write( - timestamps, - (boolean[]) valueArray, - bitMap, - arrayOffset, - pointsInSegment, - arrayOffset); - case INT32, DATE -> - valueChunkWriter - .getPageWriter() - .write( - timestamps, - (int[]) valueArray, - bitMap, - arrayOffset, - pointsInSegment, - arrayOffset); - case INT64, TIMESTAMP -> - valueChunkWriter - .getPageWriter() - .write( - timestamps, - (long[]) valueArray, - bitMap, - arrayOffset, - pointsInSegment, - arrayOffset); - case FLOAT -> - valueChunkWriter - .getPageWriter() - .write( - timestamps, - (float[]) valueArray, - bitMap, - arrayOffset, - pointsInSegment, - arrayOffset); - case DOUBLE -> - valueChunkWriter - .getPageWriter() - .write( - timestamps, - (double[]) valueArray, - bitMap, - arrayOffset, - pointsInSegment, - arrayOffset); - case TEXT, STRING, BLOB, OBJECT -> - valueChunkWriter - .getPageWriter() - .write( - timestamps, - (Binary[]) valueArray, - bitMap, - arrayOffset, - pointsInSegment, - arrayOffset); - case VECTOR, UNKNOWN -> throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + dataType); - } + TypeServices.StorageEngine.VALUE_CHUNK_ARRAY_BITMAP_WRITER_SERVICE + .call(Type.fromTsDataType(dataType)) + .write(valueChunkWriter, timestamps, valueArray, bitMap, arrayOffset, pointsInSegment); } private void handleEncodingWithDeletedMeasurements( @@ -1107,6 +956,9 @@ private void handleEncodingWithDeletedMeasurements( lastValidPointIndexForTimeDupCheck[columnIndex] = new Pair<>(Long.MIN_VALUE, null); } TSDataType tsDataType = activeSchemaList.get(columnIndex).getType(); + TypeServices.AlignedTVListColumnWriter columnWriter = + TypeServices.StorageEngine.ALIGNED_TV_LIST_COLUMN_WRITER_SERVICE.call( + Type.fromTsDataType(tsDataType)); for (int sortedRowIndex = pageRange.get(pageNum * 2); sortedRowIndex <= pageRange.get(pageNum * 2 + 1); sortedRowIndex++) { @@ -1153,68 +1005,13 @@ private void handleEncodingWithDeletedMeasurements( boolean isNull = tvListColumnIndex < 0 || alignedWorkingListForFlush.isNullValue(originRowIndex, tvListColumnIndex); - switch (tsDataType) { - case BOOLEAN: - alignedChunkWriter.writeByColumn( - time, - !isNull - && alignedWorkingListForFlush.getBooleanByValueIndex( - originRowIndex, tvListColumnIndex), - isNull); - break; - case INT32: - case DATE: - alignedChunkWriter.writeByColumn( - time, - isNull - ? 0 - : alignedWorkingListForFlush.getIntByValueIndex( - originRowIndex, tvListColumnIndex), - isNull); - break; - case INT64: - case TIMESTAMP: - alignedChunkWriter.writeByColumn( - time, - isNull - ? 0 - : alignedWorkingListForFlush.getLongByValueIndex( - originRowIndex, tvListColumnIndex), - isNull); - break; - case FLOAT: - alignedChunkWriter.writeByColumn( - time, - isNull - ? 0 - : alignedWorkingListForFlush.getFloatByValueIndex( - originRowIndex, tvListColumnIndex), - isNull); - break; - case DOUBLE: - alignedChunkWriter.writeByColumn( - time, - isNull - ? 0 - : alignedWorkingListForFlush.getDoubleByValueIndex( - originRowIndex, tvListColumnIndex), - isNull); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - alignedChunkWriter.writeByColumn( - time, - isNull - ? null - : alignedWorkingListForFlush.getBinaryByValueIndex( - originRowIndex, tvListColumnIndex), - isNull); - break; - default: - break; - } + columnWriter.write( + alignedChunkWriter, + time, + alignedWorkingListForFlush, + originRowIndex, + tvListColumnIndex, + isNull); } alignedChunkWriter.nextColumn(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java index a1d2f59f735f..bb2b5f704e5b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java @@ -41,10 +41,9 @@ import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; -import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.IPointReader; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -160,6 +159,7 @@ public void sortTvLists() { public void initChunkMetaFromTvLists(Filter globalTimeFilter) { // create chunk statistics Statistics chunkStatistics = Statistics.getStatsByType(dataType); + Type type = Type.fromTsDataType(dataType); timeValuePairIterator = createMemPointIterator(Ordering.ASC, globalTimeFilter); timeValuePairIterator.setStreamingQueryMemChunk(false); while (timeValuePairIterator.hasNextBatch()) { @@ -169,58 +169,9 @@ public void initChunkMetaFromTvLists(Filter globalTimeFilter) { TsBlock tsBlock = timeValuePairIterator.nextBatch(); if (!tsBlock.isEmpty()) { - switch (dataType) { - case BOOLEAN: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - long time = tsBlock.getTimeByIndex(i); - chunkStatistics.update(time, tsBlock.getColumn(0).getBoolean(i)); - pageStatistics.update(time, tsBlock.getColumn(0).getBoolean(i)); - } - break; - case INT32: - case DATE: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - long time = tsBlock.getTimeByIndex(i); - chunkStatistics.update(time, tsBlock.getColumn(0).getInt(i)); - pageStatistics.update(time, tsBlock.getColumn(0).getInt(i)); - } - break; - case INT64: - case TIMESTAMP: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - long time = tsBlock.getTimeByIndex(i); - chunkStatistics.update(time, tsBlock.getColumn(0).getLong(i)); - pageStatistics.update(time, tsBlock.getColumn(0).getLong(i)); - } - break; - case FLOAT: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - long time = tsBlock.getTimeByIndex(i); - chunkStatistics.update(time, tsBlock.getColumn(0).getFloat(i)); - pageStatistics.update(time, tsBlock.getColumn(0).getFloat(i)); - } - break; - case DOUBLE: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - long time = tsBlock.getTimeByIndex(i); - chunkStatistics.update(time, tsBlock.getColumn(0).getDouble(i)); - pageStatistics.update(time, tsBlock.getColumn(0).getDouble(i)); - } - break; - case TEXT: - case BLOB: - case STRING: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - long time = tsBlock.getTimeByIndex(i); - chunkStatistics.update(time, tsBlock.getColumn(0).getBinary(i)); - pageStatistics.update(time, tsBlock.getColumn(0).getBinary(i)); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - StorageEngineMessages.STORAGE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataType)); + for (int i = 0; i < tsBlock.getPositionCount(); i++) { + type.update(chunkStatistics, tsBlock, 0, i); + type.update(pageStatistics, tsBlock, 0, i); } } } @@ -331,41 +282,12 @@ private TsBlock buildTsBlock() { // read all data in memory chunk and write to tsblock private void writeValidValuesIntoTsBlock(TsBlockBuilder builder) throws IOException { MemPointIterator timeValuePairIterator = createMemPointIterator(Ordering.ASC, null); + Type type = Type.fromTsDataType(dataType); while (timeValuePairIterator.hasNextTimeValuePair()) { TimeValuePair tvPair = timeValuePairIterator.nextTimeValuePair(); builder.getTimeColumnBuilder().writeLong(tvPair.getTimestamp()); - switch (dataType) { - case BOOLEAN: - builder.getColumnBuilder(0).writeBoolean(tvPair.getValue().getBoolean()); - break; - case INT32: - case DATE: - if (builder.getColumnBuilder(0) instanceof BinaryColumnBuilder) { - ((BinaryColumnBuilder) builder.getColumnBuilder(0)) - .writeDate(tvPair.getValue().getInt()); - } else { - builder.getColumnBuilder(0).writeInt(tvPair.getValue().getInt()); - } - break; - case INT64: - case TIMESTAMP: - builder.getColumnBuilder(0).writeLong(tvPair.getValue().getLong()); - break; - case FLOAT: - builder.getColumnBuilder(0).writeFloat(tvPair.getValue().getFloat()); - break; - case DOUBLE: - builder.getColumnBuilder(0).writeDouble(tvPair.getValue().getDouble()); - break; - case TEXT: - case STRING: - case BLOB: - builder.getColumnBuilder(0).writeBinary(tvPair.getValue().getBinary()); - break; - default: - break; - } + type.write(builder.getColumnBuilder(0), tvPair.getValue()); builder.declarePosition(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java index a5a4ce25848c..7e54296c6a8b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java @@ -24,10 +24,10 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; -import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo; import org.apache.iotdb.db.utils.datastructure.MemPointIterator; import org.apache.iotdb.db.utils.datastructure.MemPointIteratorFactory; @@ -38,6 +38,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; @@ -45,8 +46,6 @@ import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.DataInputStream; import java.io.IOException; @@ -55,8 +54,6 @@ import java.util.Optional; import java.util.concurrent.BlockingQueue; -import static org.apache.iotdb.db.utils.MemUtils.getBinarySize; - public class WritableMemChunk extends AbstractWritableMemChunk { private IMeasurementSchema schema; @@ -64,10 +61,9 @@ public class WritableMemChunk extends AbstractWritableMemChunk { private TVList list; private List sortedList; private long sortedRowCount = 0; + private TypeServices.TVListObjectWriter rowWriter; private static final String UNSUPPORTED_TYPE = DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE; - private static final Logger LOGGER = LoggerFactory.getLogger(WritableMemChunk.class); - private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); private final int TVLIST_SORT_THRESHOLD = CONFIG.getTvListSortThreshold(); @@ -78,6 +74,9 @@ public WritableMemChunk(IMeasurementSchema schema) { this.schema = schema; this.list = TVList.newList(schema.getType()); this.sortedList = new ArrayList<>(); + this.rowWriter = + TypeServices.StorageEngine.TV_LIST_OBJECT_WRITER_SERVICE.call( + Type.fromTsDataType(schema.getType())); this.encryptParameter = EncryptUtils.getEncryptParameter(); } @@ -85,6 +84,9 @@ public WritableMemChunk(IMeasurementSchema schema, EncryptParameter encryptParam this.schema = schema; this.list = TVList.newList(schema.getType()); this.sortedList = new ArrayList<>(); + this.rowWriter = + TypeServices.StorageEngine.TV_LIST_OBJECT_WRITER_SERVICE.call( + Type.fromTsDataType(schema.getType())); this.encryptParameter = encryptParameter; } @@ -103,32 +105,7 @@ protected void handoverTvList() { @Override public void writeNonAlignedPoint(long insertTime, Object objectValue) { - switch (schema.getType()) { - case BOOLEAN: - putBoolean(insertTime, (boolean) objectValue); - break; - case INT32: - case DATE: - putInt(insertTime, (int) objectValue); - break; - case INT64: - case TIMESTAMP: - putLong(insertTime, (long) objectValue); - break; - case FLOAT: - putFloat(insertTime, (float) objectValue); - break; - case DOUBLE: - putDouble(insertTime, (double) objectValue); - break; - case TEXT: - case BLOB: - case STRING: - putBinary(insertTime, (Binary) objectValue); - break; - default: - throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + schema.getType().name()); - } + rowWriter.write(list, insertTime, objectValue); if (TVLIST_SORT_THRESHOLD > 0 && list.rowCount() >= TVLIST_SORT_THRESHOLD) { handoverTvList(); } @@ -143,39 +120,9 @@ public void writeAlignedPoints( @Override public void writeNonAlignedTablet( long[] times, Object valueList, BitMap bitMap, TSDataType dataType, int start, int end) { - switch (dataType) { - case BOOLEAN: - boolean[] boolValues = (boolean[]) valueList; - putBooleans(times, boolValues, bitMap, start, end); - break; - case INT32: - case DATE: - int[] intValues = (int[]) valueList; - putInts(times, intValues, bitMap, start, end); - break; - case INT64: - case TIMESTAMP: - long[] longValues = (long[]) valueList; - putLongs(times, longValues, bitMap, start, end); - break; - case FLOAT: - float[] floatValues = (float[]) valueList; - putFloats(times, floatValues, bitMap, start, end); - break; - case DOUBLE: - double[] doubleValues = (double[]) valueList; - putDoubles(times, doubleValues, bitMap, start, end); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary[] binaryValues = (Binary[]) valueList; - putBinaries(times, binaryValues, bitMap, start, end); - break; - default: - throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + dataType.name()); - } + TypeServices.StorageEngine.TV_LIST_ARRAY_WRITER_SERVICE + .call(Type.fromTsDataType(dataType)) + .write(list, times, valueList, bitMap, start, end); if (TVLIST_SORT_THRESHOLD > 0 && list.rowCount() >= TVLIST_SORT_THRESHOLD) { handoverTvList(); } @@ -388,6 +335,9 @@ public void encodeWorkingTVList( TSDataType tsDataType = schema.getType(); ChunkWriterImpl chunkWriterImpl = createIChunkWriter(); + TypeServices.TVListChunkWriter valueWriter = + TypeServices.StorageEngine.TV_LIST_CHUNK_WRITER_SERVICE.call( + Type.fromTsDataType(tsDataType)); long dataSizeInCurrentChunk = 0; int pointNumInCurrentChunk = 0; for (int sortedRowIndex = 0; @@ -410,40 +360,8 @@ public void encodeWorkingTVList( chunkWriterImpl.setLastPoint(true); } - switch (tsDataType) { - case BOOLEAN: - chunkWriterImpl.write(time, workingListForFlush.getBoolean(sortedRowIndex)); - dataSizeInCurrentChunk += 8L + 1L; - break; - case INT32: - case DATE: - chunkWriterImpl.write(time, workingListForFlush.getInt(sortedRowIndex)); - dataSizeInCurrentChunk += 8L + 4L; - break; - case INT64: - case TIMESTAMP: - chunkWriterImpl.write(time, workingListForFlush.getLong(sortedRowIndex)); - dataSizeInCurrentChunk += 8L + 8L; - break; - case FLOAT: - chunkWriterImpl.write(time, workingListForFlush.getFloat(sortedRowIndex)); - dataSizeInCurrentChunk += 8L + 4L; - break; - case DOUBLE: - chunkWriterImpl.write(time, workingListForFlush.getDouble(sortedRowIndex)); - dataSizeInCurrentChunk += 8L + 8L; - break; - case TEXT: - case BLOB: - case STRING: - Binary value = workingListForFlush.getBinary(sortedRowIndex); - chunkWriterImpl.write(time, value); - dataSizeInCurrentChunk += 8L + getBinarySize(value); - break; - default: - LOGGER.error(StorageEngineMessages.WRITABLE_MEM_CHUNK_UNSUPPORTED_TYPE, tsDataType); - break; - } + dataSizeInCurrentChunk += + valueWriter.write(chunkWriterImpl, time, workingListForFlush, sortedRowIndex); pointNumInCurrentChunk++; if (pointNumInCurrentChunk > maxNumberOfPointsInChunk || dataSizeInCurrentChunk > targetChunkSize) { @@ -548,6 +466,9 @@ public void serializeToWAL(IWALByteBufferView buffer) { public static WritableMemChunk deserialize(DataInputStream stream) throws IOException { WritableMemChunk memChunk = new WritableMemChunk(); memChunk.schema = MeasurementSchema.deserializeFrom(stream); + memChunk.rowWriter = + TypeServices.StorageEngine.TV_LIST_OBJECT_WRITER_SERVICE.call( + Type.fromTsDataType(memChunk.schema.getType())); int sortedListSize = stream.readInt(); memChunk.sortedList = new ArrayList<>(); for (int i = 0; i < sortedListSize; i++) { @@ -562,6 +483,9 @@ public static WritableMemChunk deserializeSingleTVListMemChunks(DataInputStream throws IOException { WritableMemChunk memChunk = new WritableMemChunk(); memChunk.schema = MeasurementSchema.deserializeFrom(stream); + memChunk.rowWriter = + TypeServices.StorageEngine.TV_LIST_OBJECT_WRITER_SERVICE.call( + Type.fromTsDataType(memChunk.schema.getType())); memChunk.list = TVList.deserialize(stream); return memChunk; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java index a0b8a7aa17e5..a673bb6efb14 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java @@ -31,12 +31,12 @@ import org.apache.tsfile.read.common.BatchDataFactory; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.FilterFactory; import org.apache.tsfile.read.reader.IPageReader; import org.apache.tsfile.read.reader.series.PaginationController; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -284,53 +284,9 @@ private void updatePageStatisticsFromTsBlock( // timeStatistics.setEmpty(false); for (int column = 0; column < tsDataTypes.size(); column++) { - switch (tsDataTypes.get(column)) { - case BOOLEAN: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - valueStatistics[column].update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getBoolean(i)); - } - break; - case INT32: - case DATE: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - valueStatistics[column].update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getInt(i)); - } - break; - case INT64: - case TIMESTAMP: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - valueStatistics[column].update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getLong(i)); - } - break; - case FLOAT: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - valueStatistics[column].update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getFloat(i)); - } - break; - case DOUBLE: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - valueStatistics[column].update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getDouble(i)); - } - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - valueStatistics[column].update( - tsBlock.getTimeByIndex(i), tsBlock.getColumn(column).getBinary(i)); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - StorageEngineMessages.STORAGE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - tsDataTypes.get(column))); + Type type = Type.fromTsDataType(tsDataTypes.get(column)); + for (int i = 0; i < tsBlock.getPositionCount(); i++) { + type.update(valueStatistics[column], tsBlock, column, i); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java index b7e7bf6d995a..dca6876fb623 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java @@ -19,8 +19,8 @@ package org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk; -import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.PageMetadata; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -30,11 +30,11 @@ import org.apache.tsfile.read.common.BatchDataFactory; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.FilterFactory; import org.apache.tsfile.read.reader.IPageReader; import org.apache.tsfile.read.reader.series.PaginationController; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.IOException; import java.io.Serializable; @@ -82,40 +82,14 @@ public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException { BatchData batchData = BatchDataFactory.createBatchData(tsDataType, ascending, false); boolean[] satisfyInfo = buildSatisfyInfoArray(null); + TypeServices.BatchDataColumnWriter valueWriter = + TypeServices.StorageEngine.BATCH_DATA_COLUMN_WRITER_SERVICE.call( + Type.fromTsDataType(tsDataType)); + Column valueColumn = tsBlock.getColumn(0); for (int i = 0; i < tsBlock.getPositionCount(); i++) { if (satisfyInfo[i]) { - switch (tsDataType) { - case BOOLEAN: - batchData.putBoolean( - tsBlock.getTimeColumn().getLong(i), tsBlock.getColumn(0).getBoolean(i)); - break; - case INT32: - case DATE: - batchData.putInt(tsBlock.getTimeColumn().getLong(i), tsBlock.getColumn(0).getInt(i)); - break; - case INT64: - case TIMESTAMP: - batchData.putLong(tsBlock.getTimeColumn().getLong(i), tsBlock.getColumn(0).getLong(i)); - break; - case DOUBLE: - batchData.putDouble( - tsBlock.getTimeColumn().getLong(i), tsBlock.getColumn(0).getDouble(i)); - break; - case FLOAT: - batchData.putFloat( - tsBlock.getTimeColumn().getLong(i), tsBlock.getColumn(0).getFloat(i)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - batchData.putBinary( - tsBlock.getTimeColumn().getLong(i), tsBlock.getColumn(0).getBinary(i)); - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(tsDataType)); - } + valueWriter.write(batchData, tsBlock.getTimeByIndex(i), valueColumn, i); } } return batchData.flip(); @@ -279,47 +253,9 @@ private void initPageStatistics() { private void updatePageStatisticsFromTsBlock(Statistics statistics) { if (!tsBlock.isEmpty()) { - switch (tsDataType) { - case BOOLEAN: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - statistics.update(tsBlock.getTimeByIndex(i), tsBlock.getColumn(0).getBoolean(i)); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - statistics.update(tsBlock.getTimeByIndex(i), tsBlock.getColumn(0).getBinary(i)); - } - break; - case FLOAT: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - statistics.update(tsBlock.getTimeByIndex(i), tsBlock.getColumn(0).getFloat(i)); - } - break; - case INT32: - case DATE: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - statistics.update(tsBlock.getTimeByIndex(i), tsBlock.getColumn(0).getInt(i)); - } - break; - case INT64: - case TIMESTAMP: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - statistics.update(tsBlock.getTimeByIndex(i), tsBlock.getColumn(0).getLong(i)); - } - break; - case DOUBLE: - for (int i = 0; i < tsBlock.getPositionCount(); i++) { - statistics.update(tsBlock.getTimeByIndex(i), tsBlock.getColumn(0).getDouble(i)); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - StorageEngineMessages.STORAGE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - tsDataType)); + Type type = Type.fromTsDataType(tsDataType); + for (int i = 0; i < tsBlock.getPositionCount(); i++) { + type.update(statistics, tsBlock, 0, i); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java index 755fe88bb46a..1a7785563126 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java @@ -47,6 +47,7 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.page.PageReader; import org.apache.tsfile.read.reader.page.TimePageReader; import org.apache.tsfile.read.reader.page.ValuePageReader; @@ -470,11 +471,11 @@ public static void updateTsFileResource( if (timeseriesMetaData.getTsDataType() != TSDataType.BLOB) { TsPrimitiveType value; value = - TsPrimitiveType.getByType( - timeseriesMetaData.getTsDataType() == TSDataType.VECTOR - ? TSDataType.INT64 - : timeseriesMetaData.getTsDataType(), - timeseriesMetaData.getStatistics().getLastValue()); + Type.fromTsDataType( + timeseriesMetaData.getTsDataType() == TSDataType.VECTOR + ? TSDataType.INT64 + : timeseriesMetaData.getTsDataType()) + .getTsPrimitiveType(timeseriesMetaData.getStatistics().getLastValue()); seriesLastValues.add( new Pair<>( timeseriesMetaData.getMeasurementId(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 1be41760e8e3..257e057045a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -68,6 +68,7 @@ import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; @@ -677,11 +678,11 @@ private void endTsFileResource( TsPrimitiveType lastValue = chunkMetadata.getStatistics() != null && chunkMetadata.getDataType() != TSDataType.BLOB - ? TsPrimitiveType.getByType( - chunkMetadata.getDataType() == TSDataType.VECTOR - ? TSDataType.INT64 - : chunkMetadata.getDataType(), - chunkMetadata.getStatistics().getLastValue()) + ? Type.fromTsDataType( + chunkMetadata.getDataType() == TSDataType.VECTOR + ? TSDataType.INT64 + : chunkMetadata.getDataType()) + .getTsPrimitiveType(chunkMetadata.getStatistics().getLastValue()) : null; TimeValuePair timeValuePair = lastValue != null diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java index f7350c87323a..d8a8e6adce93 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java @@ -21,8 +21,8 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.utils.TimePartitionUtils; -import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.PageException; @@ -34,12 +34,12 @@ import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.Chunk; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.apache.tsfile.write.writer.TsFileIOWriter; @@ -60,13 +60,6 @@ import static org.apache.iotdb.db.storageengine.load.LoadTsFileManager.MEASUREMENT_ID_CACHE; public class AlignedChunkData implements ChunkData { - protected static final int DEFAULT_INT32 = 0; - protected static final long DEFAULT_INT64 = 0L; - protected static final float DEFAULT_FLOAT = 0; - protected static final double DEFAULT_DOUBLE = 0.0; - protected static final boolean DEFAULT_BOOLEAN = false; - protected static final Binary DEFAULT_BINARY = null; - protected final TTimePartitionSlot timePartitionSlot; protected final IDeviceID device; protected List chunkHeaderList; @@ -232,6 +225,9 @@ public void writeDecodePage(final long[] times, final Object[] values, final int public void writeDecodeValuePage( final long[] times, final TsPrimitiveType[] values, final TSDataType dataType) throws IOException { + final Type type = Type.fromTsDataType(dataType); + final TypeServices.ValueSerializer valueSerializer = + TypeServices.StorageEngine.TS_PRIMITIVE_VALUE_SERIALIZER_SERVICE.call(type); pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); final long startTime = timePartitionSlot.getStartTime(); final long endTime = TimePartitionUtils.getTimePartitionEndTime(startTime); @@ -250,35 +246,7 @@ public void writeDecodeValuePage( dataSize += ReadWriteIOUtils.write(true, stream); } else { dataSize += ReadWriteIOUtils.write(false, stream); - switch (dataType) { - case INT32: - case DATE: - dataSize += ReadWriteIOUtils.write(values[i].getInt(), stream); - break; - case INT64: - case TIMESTAMP: - dataSize += ReadWriteIOUtils.write(values[i].getLong(), stream); - break; - case FLOAT: - dataSize += ReadWriteIOUtils.write(values[i].getFloat(), stream); - break; - case DOUBLE: - dataSize += ReadWriteIOUtils.write(values[i].getDouble(), stream); - break; - case BOOLEAN: - dataSize += ReadWriteIOUtils.write(values[i].getBoolean(), stream); - break; - case TEXT: - case BLOB: - case STRING: - dataSize += ReadWriteIOUtils.write(values[i].getBinary(), stream); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - StorageEngineMessages.STORAGE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataType)); - } + dataSize += valueSerializer.serialize(values[i], stream); } } } @@ -367,6 +335,13 @@ private void buildChunk( throws IOException, PageException { boolean needDecode; int decodePageIndex = 0; + final Type type = isTimeChunk ? null : Type.fromTsDataType(chunkHeader.getDataType()); + final ValueChunkWriter valueChunkWriter = + isTimeChunk ? null : chunkWriter.getValueChunkWriterByIndex(valueChunkIndex); + final TypeServices.DecodedValueChunkWriter decodedValueWriter = + isTimeChunk + ? null + : TypeServices.StorageEngine.DECODED_VALUE_CHUNK_WRITER_SERVICE.call(type); for (int j = 0; j < pageNumber; j++) { needDecode = ReadWriteIOUtils.readBool(stream); if (needDecode) { @@ -382,46 +357,7 @@ private void buildChunk( chunkWriter.writeTime(time); } else { final boolean isNull = ReadWriteIOUtils.readBool(stream); - switch (chunkHeader.getDataType()) { - case INT32: - case DATE: - final int int32Value = isNull ? DEFAULT_INT32 : ReadWriteIOUtils.readInt(stream); - chunkWriter.write(timePageBatch[i], int32Value, isNull, valueChunkIndex); - break; - case INT64: - case TIMESTAMP: - final long int64Value = isNull ? DEFAULT_INT64 : ReadWriteIOUtils.readLong(stream); - chunkWriter.write(timePageBatch[i], int64Value, isNull, valueChunkIndex); - break; - case FLOAT: - final float floatValue = - isNull ? DEFAULT_FLOAT : ReadWriteIOUtils.readFloat(stream); - chunkWriter.write(timePageBatch[i], floatValue, isNull, valueChunkIndex); - break; - case DOUBLE: - final double doubleValue = - isNull ? DEFAULT_DOUBLE : ReadWriteIOUtils.readDouble(stream); - chunkWriter.write(timePageBatch[i], doubleValue, isNull, valueChunkIndex); - break; - case BOOLEAN: - final boolean boolValue = - isNull ? DEFAULT_BOOLEAN : ReadWriteIOUtils.readBool(stream); - chunkWriter.write(timePageBatch[i], boolValue, isNull, valueChunkIndex); - break; - case TEXT: - case BLOB: - case STRING: - final Binary binaryValue = - isNull ? DEFAULT_BINARY : ReadWriteIOUtils.readBinary(stream); - chunkWriter.write(timePageBatch[i], binaryValue, isNull, valueChunkIndex); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - StorageEngineMessages - .STORAGE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - chunkHeader.getDataType())); - } + decodedValueWriter.write(valueChunkWriter, timePageBatch[i], stream, isNull); } } if (isTimeChunk) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java index 998441da3ae9..d2df450b1393 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java @@ -21,7 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.utils.TimePartitionUtils; -import org.apache.iotdb.db.i18n.StorageEngineMessages; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.PageException; @@ -29,10 +29,9 @@ import org.apache.tsfile.file.header.PageHeader; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.statistics.Statistics; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.schema.MeasurementSchema; import org.apache.tsfile.write.writer.TsFileIOWriter; @@ -63,6 +62,9 @@ public BatchedAlignedValueChunkData(IDeviceID device, TTimePartitionSlot timePar @Override public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataType dataType) throws IOException { + final Type type = Type.fromTsDataType(dataType); + final TypeServices.ValueSerializer valueSerializer = + TypeServices.StorageEngine.TS_PRIMITIVE_VALUE_SERIALIZER_SERVICE.call(type); pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); final long startTime = timePartitionSlot.getStartTime(); final long endTime = TimePartitionUtils.getTimePartitionEndTime(startTime); @@ -86,36 +88,7 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT pageStartTime = Math.min(pageStartTime, times[i]); pageEndTime = Math.max(pageEndTime, times[i]); dataSize += ReadWriteIOUtils.write(false, stream); - switch (dataType) { - case INT32: - case DATE: - dataSize += ReadWriteIOUtils.write(values[i].getInt(), stream); - break; - case INT64: - case TIMESTAMP: - dataSize += ReadWriteIOUtils.write(values[i].getLong(), stream); - break; - case FLOAT: - dataSize += ReadWriteIOUtils.write(values[i].getFloat(), stream); - break; - case DOUBLE: - dataSize += ReadWriteIOUtils.write(values[i].getDouble(), stream); - break; - case BOOLEAN: - dataSize += ReadWriteIOUtils.write(values[i].getBoolean(), stream); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - dataSize += ReadWriteIOUtils.write(values[i].getBinary(), stream); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - StorageEngineMessages.STORAGE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataType)); - } + dataSize += valueSerializer.serialize(values[i], stream); } } } @@ -171,46 +144,12 @@ private void buildValueChunkWriter( // ValueChunkWriter, first use the wrong time. Before sealing each page, modify the statistic // in the page writer and set the correct start time and end time. final int length = ReadWriteIOUtils.readInt(stream); + final TypeServices.DecodedValueChunkWriter decodedValueWriter = + TypeServices.StorageEngine.DECODED_VALUE_CHUNK_WRITER_SERVICE.call( + Type.fromTsDataType(chunkHeader.getDataType())); for (int j = 0; j < length; j++) { final boolean isNull = ReadWriteIOUtils.readBool(stream); - switch (chunkHeader.getDataType()) { - case INT32: - case DATE: - final int int32Value = isNull ? DEFAULT_INT32 : ReadWriteIOUtils.readInt(stream); - valueChunkWriter.write(0, int32Value, isNull); - break; - case INT64: - case TIMESTAMP: - final long int64Value = isNull ? DEFAULT_INT64 : ReadWriteIOUtils.readLong(stream); - valueChunkWriter.write(0, int64Value, isNull); - break; - case FLOAT: - final float floatValue = isNull ? DEFAULT_FLOAT : ReadWriteIOUtils.readFloat(stream); - valueChunkWriter.write(0, floatValue, isNull); - break; - case DOUBLE: - final double doubleValue = - isNull ? DEFAULT_DOUBLE : ReadWriteIOUtils.readDouble(stream); - valueChunkWriter.write(0, doubleValue, isNull); - break; - case BOOLEAN: - final boolean boolValue = isNull ? DEFAULT_BOOLEAN : ReadWriteIOUtils.readBool(stream); - valueChunkWriter.write(0, boolValue, isNull); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - final Binary binaryValue = - isNull ? DEFAULT_BINARY : ReadWriteIOUtils.readBinary(stream); - valueChunkWriter.write(0, binaryValue, isNull); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - StorageEngineMessages.STORAGE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - chunkHeader.getDataType())); - } + decodedValueWriter.write(valueChunkWriter, 0, stream, isNull); } Statistics statistics = valueChunkWriter.getPageWriter().getStatistics(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java index 7c385c3107bf..34f91385f03f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java @@ -21,8 +21,8 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.utils.TimePartitionUtils; -import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.ChunkHeader; @@ -33,10 +33,9 @@ import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.Chunk; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.schema.MeasurementSchema; import org.apache.tsfile.write.writer.TsFileIOWriter; @@ -185,6 +184,9 @@ public void writeDecodePage(final long[] times, final Object[] values, final int final long endTime = TimePartitionUtils.getTimePartitionEndTime(startTime); dataSize += ReadWriteIOUtils.write(true, stream); dataSize += ReadWriteIOUtils.write(satisfiedLength, stream); + final Type type = Type.fromTsDataType(chunkHeader.getDataType()); + final TypeServices.ValueSerializer valueSerializer = + TypeServices.StorageEngine.OBJECT_VALUE_SERIALIZER_SERVICE.call(type); for (int i = 0; i < times.length; i++) { if (times[i] > endTime) { @@ -192,35 +194,7 @@ public void writeDecodePage(final long[] times, final Object[] values, final int } if (times[i] >= startTime) { dataSize += ReadWriteIOUtils.write(times[i], stream); - switch (chunkHeader.getDataType()) { - case INT32: - case DATE: - dataSize += ReadWriteIOUtils.write((int) values[i], stream); - break; - case INT64: - case TIMESTAMP: - dataSize += ReadWriteIOUtils.write((long) values[i], stream); - break; - case FLOAT: - dataSize += ReadWriteIOUtils.write((float) values[i], stream); - break; - case DOUBLE: - dataSize += ReadWriteIOUtils.write((double) values[i], stream); - break; - case BOOLEAN: - dataSize += ReadWriteIOUtils.write((boolean) values[i], stream); - break; - case TEXT: - case BLOB: - case STRING: - dataSize += ReadWriteIOUtils.write((Binary) values[i], stream); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - StorageEngineMessages.STORAGE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - chunkHeader.getDataType())); - } + dataSize += valueSerializer.serialize(values[i], stream); } } } @@ -250,41 +224,16 @@ private void buildChunkWriter(final InputStream stream) throws IOException, Page chunkHeader.getEncodingType(), chunkHeader.getCompressionType())); boolean needDecode; + final TypeServices.DecodedChunkWriter decodedChunkWriter = + TypeServices.StorageEngine.DECODED_CHUNK_WRITER_SERVICE.call( + Type.fromTsDataType(chunkHeader.getDataType())); for (int j = 0; j < pageNumber; j++) { needDecode = ReadWriteIOUtils.readBool(stream); if (needDecode) { final int length = ReadWriteIOUtils.readInt(stream); for (int i = 0; i < length; i++) { final long time = ReadWriteIOUtils.readLong(stream); - switch (chunkHeader.getDataType()) { - case INT32: - case DATE: - chunkWriter.write(time, ReadWriteIOUtils.readInt(stream)); - break; - case INT64: - case TIMESTAMP: - chunkWriter.write(time, ReadWriteIOUtils.readLong(stream)); - break; - case FLOAT: - chunkWriter.write(time, ReadWriteIOUtils.readFloat(stream)); - break; - case DOUBLE: - chunkWriter.write(time, ReadWriteIOUtils.readDouble(stream)); - break; - case BOOLEAN: - chunkWriter.write(time, ReadWriteIOUtils.readBool(stream)); - break; - case TEXT: - case BLOB: - case STRING: - chunkWriter.write(time, ReadWriteIOUtils.readBinary(stream)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - StorageEngineMessages.STORAGE_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - chunkHeader.getDataType())); - } + decodedChunkWriter.write(chunkWriter, time, stream); } chunkWriter.sealCurrentPage(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java index cb996fd25e1d..f46fd6de554b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java @@ -25,17 +25,21 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.service.metrics.memory.StorageEngineMemoryMetrics; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.db.utils.datastructure.TVListSortAlgorithm; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.reflect.Array; import java.util.ArrayDeque; import java.util.Arrays; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.IntFunction; /** Manage all primitive data lists in memory, including get and release operations. */ public class PrimitiveArrayManager { @@ -230,36 +234,9 @@ private static void updateLimits() { } private static Object createPrimitiveArray(TSDataType dataType) { - Object dataArray; - switch (dataType) { - case BOOLEAN: - dataArray = new boolean[ARRAY_SIZE]; - break; - case INT32: - case DATE: - dataArray = new int[ARRAY_SIZE]; - break; - case INT64: - case TIMESTAMP: - dataArray = new long[ARRAY_SIZE]; - break; - case FLOAT: - dataArray = new float[ARRAY_SIZE]; - break; - case DOUBLE: - dataArray = new double[ARRAY_SIZE]; - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - dataArray = new Binary[ARRAY_SIZE]; - break; - default: - throw new UnSupportedDataTypeException(dataType.name()); - } - - return dataArray; + return TypeServices.StorageEngine.PRIMITIVE_ARRAY_ALLOCATOR_SERVICE + .call(Type.fromTsDataType(dataType)) + .apply(ARRAY_SIZE); } /** @@ -310,51 +287,13 @@ public static void close() { */ public static Object createDataListsByType(TSDataType dataType, int size) { int arrayNumber = getArrayRowCount(size); - switch (dataType) { - case BOOLEAN: - boolean[][] booleans = new boolean[arrayNumber][]; - for (int i = 0; i < arrayNumber; i++) { - booleans[i] = new boolean[ARRAY_SIZE]; - } - return booleans; - case INT32: - case DATE: - int[][] ints = new int[arrayNumber][]; - for (int i = 0; i < arrayNumber; i++) { - ints[i] = new int[ARRAY_SIZE]; - } - return ints; - case INT64: - case TIMESTAMP: - long[][] longs = new long[arrayNumber][]; - for (int i = 0; i < arrayNumber; i++) { - longs[i] = new long[ARRAY_SIZE]; - } - return longs; - case FLOAT: - float[][] floats = new float[arrayNumber][]; - for (int i = 0; i < arrayNumber; i++) { - floats[i] = new float[ARRAY_SIZE]; - } - return floats; - case DOUBLE: - double[][] doubles = new double[arrayNumber][]; - for (int i = 0; i < arrayNumber; i++) { - doubles[i] = new double[ARRAY_SIZE]; - } - return doubles; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - Binary[][] binaries = new Binary[arrayNumber][]; - for (int i = 0; i < arrayNumber; i++) { - binaries[i] = new Binary[ARRAY_SIZE]; - } - return binaries; - default: - throw new UnSupportedDataTypeException(dataType.name()); - } + IntFunction arrayAllocator = + TypeServices.StorageEngine.PRIMITIVE_ARRAY_ALLOCATOR_SERVICE.call( + Type.fromTsDataType(dataType)); + Object[] dataLists = + (Object[]) Array.newInstance(arrayAllocator.apply(0).getClass(), arrayNumber); + Arrays.setAll(dataLists, index -> arrayAllocator.apply(ARRAY_SIZE)); + return dataLists; } public static int getArrayRowCount(int size) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java index 0b27de77b123..30767c82baa0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java @@ -46,7 +46,7 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.write.record.Tablet; @@ -714,38 +714,7 @@ private void addValueToTablet( final int columnIndex, final TSDataType dataType, final Object value) { - switch (dataType) { - case BOOLEAN: - ((boolean[]) tablet.getValues()[columnIndex])[rowIndex] = (boolean) value; - break; - case INT32: - ((int[]) tablet.getValues()[columnIndex])[rowIndex] = (int) value; - break; - case DATE: - ((LocalDate[]) tablet.getValues()[columnIndex])[rowIndex] = - value instanceof LocalDate - ? (LocalDate) value - : DateUtils.parseIntToLocalDate((int) value); - break; - case INT64: - case TIMESTAMP: - ((long[]) tablet.getValues()[columnIndex])[rowIndex] = (long) value; - break; - case FLOAT: - ((float[]) tablet.getValues()[columnIndex])[rowIndex] = (float) value; - break; - case DOUBLE: - ((double[]) tablet.getValues()[columnIndex])[rowIndex] = (double) value; - break; - case TEXT: - case BLOB: - case STRING: - ((Binary[]) tablet.getValues()[columnIndex])[rowIndex] = (Binary) value; - break; - default: - LOGGER.warn(DataNodePipeMessages.PIPE_LOG_UNSUPPORTED_DATA_TYPE_C8929F11, dataType); - return; - } + Type.fromTsDataType(dataType).addValue(rowIndex, value, tablet.getValues()[columnIndex]); // Unmark the bitmap position to indicate this value is NOT null. // addTimestamp() triggers initBitMapsWithApiUsage() which marks all positions as null. final BitMap[] bitMaps = tablet.getBitMaps(); @@ -762,45 +731,9 @@ private void copyColumnValue( final TSDataType dataType, final Object sourceColumn, final int sourceRowIndex) { - switch (dataType) { - case BOOLEAN: - ((boolean[]) tablet.getValues()[targetColumnIndex])[targetRowIndex] = - ((boolean[]) sourceColumn)[sourceRowIndex]; - break; - case INT32: - ((int[]) tablet.getValues()[targetColumnIndex])[targetRowIndex] = - ((int[]) sourceColumn)[sourceRowIndex]; - break; - case DATE: - ((LocalDate[]) tablet.getValues()[targetColumnIndex])[targetRowIndex] = - sourceColumn instanceof LocalDate[] - ? ((LocalDate[]) sourceColumn)[sourceRowIndex] - : DateUtils.parseIntToLocalDate(((int[]) sourceColumn)[sourceRowIndex]); - break; - case INT64: - case TIMESTAMP: - ((long[]) tablet.getValues()[targetColumnIndex])[targetRowIndex] = - ((long[]) sourceColumn)[sourceRowIndex]; - break; - case FLOAT: - ((float[]) tablet.getValues()[targetColumnIndex])[targetRowIndex] = - ((float[]) sourceColumn)[sourceRowIndex]; - break; - case DOUBLE: - ((double[]) tablet.getValues()[targetColumnIndex])[targetRowIndex] = - ((double[]) sourceColumn)[sourceRowIndex]; - break; - case TEXT: - case BLOB: - case STRING: - ((Binary[]) tablet.getValues()[targetColumnIndex])[targetRowIndex] = - ((Binary[]) sourceColumn)[sourceRowIndex]; - break; - default: - LOGGER.warn( - DataNodePipeMessages.PIPE_LOG_UNSUPPORTED_DATA_TYPE_FOR_COPY_8AD25FE7, dataType); - return; - } + Type.fromTsDataType(dataType) + .copyArrayElement( + sourceColumn, sourceRowIndex, tablet.getValues()[targetColumnIndex], targetRowIndex); // Unmark the bitmap position to indicate this value is NOT null. final BitMap[] bitMaps = tablet.getBitMaps(); if (bitMaps != null && bitMaps[targetColumnIndex] != null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java index f36c461f9b65..9dae1181295c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java @@ -44,9 +44,8 @@ import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.page.PageReader; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.IChunkWriter; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -404,44 +403,20 @@ protected void rewritePageIntoFiles( BatchData batchData, MeasurementSchema schema, Map partitionChunkWriterMap) { + final Type type = Type.fromTsDataType(schema.getType()); + long previousPartitionId = 0; + ChunkWriterImpl chunkWriter = null; while (batchData.hasCurrent()) { long time = batchData.currentTime(); - Object value = batchData.currentValue(); long partitionId = TimePartitionUtils.getTimePartitionId(time); - ChunkWriterImpl chunkWriter = - partitionChunkWriterMap.computeIfAbsent(partitionId, v -> new ChunkWriterImpl(schema)); - getOrDefaultTsFileIOWriter(oldTsFile, partitionId); - switch (schema.getType()) { - case INT32: - case DATE: - chunkWriter.write(time, (int) value); - break; - case INT64: - case TIMESTAMP: - chunkWriter.write(time, (long) value); - break; - case FLOAT: - chunkWriter.write(time, (float) value); - break; - case DOUBLE: - chunkWriter.write(time, (double) value); - break; - case BOOLEAN: - chunkWriter.write(time, (boolean) value); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - chunkWriter.write(time, (Binary) value); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - schema.getType())); + if (chunkWriter == null || partitionId != previousPartitionId) { + previousPartitionId = partitionId; + chunkWriter = + partitionChunkWriterMap.computeIfAbsent(partitionId, v -> new ChunkWriterImpl(schema)); + getOrDefaultTsFileIOWriter(oldTsFile, partitionId); } + type.write(chunkWriter, time, batchData); batchData.next(); } partitionChunkWriterMap diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitTool.java index 2efc26a0b5a7..e5fafca708bb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitTool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitTool.java @@ -42,9 +42,8 @@ import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.page.PageReader; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.schema.MeasurementSchema; import org.apache.tsfile.write.writer.TsFileIOWriter; @@ -240,35 +239,7 @@ public void run() throws IOException { private void writeToChunkWriter( ChunkWriterImpl chunkWriter, long time, Object value, TSDataType dataType) { - switch (dataType) { - case INT32: - case DATE: - chunkWriter.write(time, (int) value); - break; - case INT64: - case TIMESTAMP: - chunkWriter.write(time, (long) value); - break; - case FLOAT: - chunkWriter.write(time, (float) value); - break; - case DOUBLE: - chunkWriter.write(time, (double) value); - break; - case BOOLEAN: - chunkWriter.write(time, (boolean) value); - break; - case TEXT: - case BLOB: - case STRING: - chunkWriter.write(time, (Binary) value); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataType)); - } + Type.fromTsDataType(dataType).write(chunkWriter, time, value); } private TsFileResource endFileAndGenerateResource(TsFileIOWriter writer) throws IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java index 41ea92670cc9..ff123e1f4e36 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java @@ -51,20 +51,19 @@ import io.airlift.airline.ParseOptionMissingException; import io.airlift.airline.ParseOptionMissingValueException; import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.lang3.StringUtils; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.TimeColumn; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.read.common.type.Type; -import java.time.LocalDate; import java.time.ZoneId; import java.util.List; import java.util.concurrent.TimeUnit; +import static org.apache.iotdb.db.utils.TypeServices.ValueConversion.VALUE_PARSER_SERVICE; + @SuppressWarnings("java:S106") // for console outputs public class CommonUtils { @@ -84,113 +83,10 @@ public static Object parseValue(TSDataType dataType, String value, ZoneId zoneId if ("null".equals(value) || "NULL".equals(value)) { return null; } - switch (dataType) { - case BOOLEAN: - return parseBoolean(value); - case INT32: - try { - return Integer.parseInt(StringUtils.trim(value)); - } catch (NumberFormatException e) { - throw new NumberFormatException( - String.format( - DataNodeMiscMessages - .MISC_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0, - value, - dataType)); - } - case INT64: - try { - return Long.parseLong(StringUtils.trim(value)); - } catch (NumberFormatException e) { - throw new NumberFormatException( - String.format( - DataNodeMiscMessages - .MISC_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0, - value, - dataType)); - } - case TIMESTAMP: - try { - if (TypeInferenceUtils.isNumber(value)) { - return Long.parseLong(value); - } else { - return DataNodeDateTimeUtils.parseDateTimeExpressionToLong( - StringUtils.trim(value), zoneId); - } - } catch (Throwable e) { - throw new NumberFormatException( - String.format( - DataNodeMiscMessages - .MISC_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_BECAUSE_50C4BF31, - value, - dataType, - e.getMessage())); - } - case DATE: - return parseIntFromString(value); - case FLOAT: - float f; - try { - f = Float.parseFloat(value); - } catch (NumberFormatException e) { - throw new NumberFormatException( - String.format( - DataNodeMiscMessages - .MISC_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0, - value, - dataType)); - } - if (Float.isInfinite(f)) { - throw new NumberFormatException(DataNodeMiscMessages.INPUT_FLOAT_INFINITY); - } - return f; - case DOUBLE: - double d; - try { - d = Double.parseDouble(value); - } catch (NumberFormatException e) { - throw new NumberFormatException( - String.format( - DataNodeMiscMessages - .MISC_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0, - value, - dataType)); - } - if (Double.isInfinite(d)) { - throw new NumberFormatException(DataNodeMiscMessages.INPUT_DOUBLE_INFINITY); - } - return d; - case TEXT: - case STRING: - if ((value.startsWith(SqlConstant.QUOTE) && value.endsWith(SqlConstant.QUOTE)) - || (value.startsWith(SqlConstant.DQUOTE) && value.endsWith(SqlConstant.DQUOTE))) { - if (value.length() == 1) { - return new Binary(value, TSFileConfig.STRING_CHARSET); - } else { - return new Binary( - value.substring(1, value.length() - 1), TSFileConfig.STRING_CHARSET); - } - } - return new Binary(value, TSFileConfig.STRING_CHARSET); - case BLOB: - if ((value.startsWith(SqlConstant.QUOTE) && value.endsWith(SqlConstant.QUOTE)) - || (value.startsWith(SqlConstant.DQUOTE) && value.endsWith(SqlConstant.DQUOTE))) { - if (value.length() == 1) { - return new Binary(parseBlobStringToByteArray(value)); - } else { - return new Binary(parseBlobStringToByteArray(value.substring(1, value.length() - 1))); - } - } - return new Binary(parseBlobStringToByteArray(value)); - case OBJECT: - throw new NumberFormatException( - String.format( - DataNodeMiscMessages - .MISC_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0, - value, - dataType)); - default: - throw new QueryProcessException(DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + dataType); + try { + return VALUE_PARSER_SERVICE.call(Type.fromTsDataType(dataType)).parse(value, zoneId); + } catch (final UnsupportedOperationException ignored) { + throw new QueryProcessException(DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + dataType); } } catch (NumberFormatException e) { throw new QueryProcessException(e.getMessage()); @@ -217,7 +113,7 @@ public static Integer parseIntFromString(String value) { } } - private static boolean parseBoolean(String value) throws QueryProcessException { + static boolean parseBoolean(String value) throws QueryProcessException { value = value.toLowerCase(); if (SqlConstant.BOOLEAN_FALSE_NUM.equals(value) || SqlConstant.BOOLEAN_FALSE.equals(value)) { return false; @@ -403,40 +299,9 @@ public static long getTTLLowerBound(long dataTTL) { public static Object createValueColumnOfDataType( TSDataType dataType, TsTableColumnCategory columnCategory, int rowNum) { - Object valueColumn; - switch (dataType) { - case INT32: - valueColumn = new int[rowNum]; - break; - case INT64: - case TIMESTAMP: - valueColumn = new long[rowNum]; - break; - case FLOAT: - valueColumn = new float[rowNum]; - break; - case DOUBLE: - valueColumn = new double[rowNum]; - break; - case BOOLEAN: - valueColumn = new boolean[rowNum]; - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - valueColumn = new Binary[rowNum]; - break; - case DATE: - valueColumn = new LocalDate[rowNum]; - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataType)); - } - return valueColumn; + return TypeServices.StorageEngine.TABLET_COLUMN_ALLOCATOR_SERVICE + .call(Type.fromTsDataType(dataType)) + .apply(rowNum); } public static void swapArray(Object[] array, int i, int j) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java index 6c26d7197fb3..5f9cbcf32472 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java @@ -21,11 +21,10 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.read.common.type.Type; public class EncodingInferenceUtils { @@ -36,29 +35,8 @@ private EncodingInferenceUtils() { /** Get default encoding by dataType */ public static TSEncoding getDefaultEncoding(TSDataType dataType) { IoTDBConfig conf = IoTDBDescriptor.getInstance().getConfig(); - switch (dataType) { - case BOOLEAN: - return conf.getDefaultBooleanEncoding(); - case DATE: - case INT32: - return conf.getDefaultInt32Encoding(); - case TIMESTAMP: - case INT64: - return conf.getDefaultInt64Encoding(); - case FLOAT: - return conf.getDefaultFloatEncoding(); - case DOUBLE: - return conf.getDefaultDoubleEncoding(); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return conf.getDefaultTextEncoding(); - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataType)); - } + return TypeServices.StorageEngine.DEFAULT_ENCODING_SERVICE + .call(Type.fromTsDataType(dataType)) + .apply(conf); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java index e87558ec669b..bb40b38dc24b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.utils; import org.apache.iotdb.commons.exception.IoTDBException; -import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; @@ -29,7 +28,7 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.Pair; @@ -125,134 +124,8 @@ public static TSQueryDataSet convertTsBlockByFetchSize(List tsBlocks) } int currentCount = tsBlock.getPositionCount(); - // serialize time column - for (int i = 0; i < currentCount; i++) { - // use columnOutput to write byte array - dataOutputStreams[0].writeLong(tsBlock.getTimeByIndex(i)); - } - - // serialize each value column and its bitmap - for (int k = 0; k < columnNum; k++) { - // get DataOutputStream for current value column and its bitmap - DataOutputStream dataOutputStream = dataOutputStreams[2 * k + 1]; - DataOutputStream dataBitmapOutputStream = dataOutputStreams[2 * (k + 1)]; - - Column column = tsBlock.getColumn(k); - TSDataType type = column.getDataType(); - switch (type) { - case INT32: - case DATE: - for (int i = 0; i < currentCount; i++) { - rowCount++; - if (column.isNull(i)) { - bitmaps[k] = bitmaps[k] << 1; - } else { - bitmaps[k] = (bitmaps[k] << 1) | FLAG; - dataOutputStream.writeInt(column.getInt(i)); - } - if (rowCount != 0 && rowCount % 8 == 0) { - dataBitmapOutputStream.writeByte(bitmaps[k]); - // we should clear the bitmap every 8 points - bitmaps[k] = 0; - } - } - break; - case INT64: - case TIMESTAMP: - for (int i = 0; i < currentCount; i++) { - rowCount++; - if (column.isNull(i)) { - bitmaps[k] = bitmaps[k] << 1; - } else { - bitmaps[k] = (bitmaps[k] << 1) | FLAG; - dataOutputStream.writeLong(column.getLong(i)); - } - if (rowCount != 0 && rowCount % 8 == 0) { - dataBitmapOutputStream.writeByte(bitmaps[k]); - // we should clear the bitmap every 8 points - bitmaps[k] = 0; - } - } - break; - case FLOAT: - for (int i = 0; i < currentCount; i++) { - rowCount++; - if (column.isNull(i)) { - bitmaps[k] = bitmaps[k] << 1; - } else { - bitmaps[k] = (bitmaps[k] << 1) | FLAG; - dataOutputStream.writeFloat(column.getFloat(i)); - } - if (rowCount != 0 && rowCount % 8 == 0) { - dataBitmapOutputStream.writeByte(bitmaps[k]); - // we should clear the bitmap every 8 points - bitmaps[k] = 0; - } - } - break; - case DOUBLE: - for (int i = 0; i < currentCount; i++) { - rowCount++; - if (column.isNull(i)) { - bitmaps[k] = bitmaps[k] << 1; - } else { - bitmaps[k] = (bitmaps[k] << 1) | FLAG; - dataOutputStream.writeDouble(column.getDouble(i)); - } - if (rowCount != 0 && rowCount % 8 == 0) { - dataBitmapOutputStream.writeByte(bitmaps[k]); - // we should clear the bitmap every 8 points - bitmaps[k] = 0; - } - } - break; - case BOOLEAN: - for (int i = 0; i < currentCount; i++) { - rowCount++; - if (column.isNull(i)) { - bitmaps[k] = bitmaps[k] << 1; - } else { - bitmaps[k] = (bitmaps[k] << 1) | FLAG; - dataOutputStream.writeBoolean(column.getBoolean(i)); - } - if (rowCount != 0 && rowCount % 8 == 0) { - dataBitmapOutputStream.writeByte(bitmaps[k]); - // we should clear the bitmap every 8 points - bitmaps[k] = 0; - } - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - for (int i = 0; i < currentCount; i++) { - rowCount++; - if (column.isNull(i)) { - bitmaps[k] = bitmaps[k] << 1; - } else { - bitmaps[k] = (bitmaps[k] << 1) | FLAG; - Binary binary = column.getBinary(i); - dataOutputStream.writeInt(binary.getLength()); - dataOutputStream.write(binary.getValues()); - } - if (rowCount != 0 && rowCount % 8 == 0) { - dataBitmapOutputStream.writeByte(bitmaps[k]); - // we should clear the bitmap every 8 points - bitmaps[k] = 0; - } - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - type)); - } - if (k != columnNum - 1) { - rowCount -= currentCount; - } - } + serializeTsBlock(rowCount, currentCount, tsBlock, columnNum, dataOutputStreams, bitmaps); + rowCount += currentCount; } // feed the remaining bitmap int remaining = rowCount % 8; @@ -299,162 +172,11 @@ private static void serializeTsBlock( DataOutputStream dataBitmapOutputStream = dataOutputStreams[2 * (k + 1)]; Column column = tsBlock.getColumn(k); - TSDataType type = column.getDataType(); - switch (type) { - case INT32: - case DATE: - doWithInt32Column(rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); - break; - case INT64: - case TIMESTAMP: - doWithInt64Column(rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); - break; - case FLOAT: - doWithFloatColumn(rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); - break; - case DOUBLE: - doWithDoubleColumn( - rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); - break; - case BOOLEAN: - doWithBooleanColumn( - rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - doWithTextColumn(rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, type)); - } - } - } - - private static void doWithInt32Column( - int rowCount, - Column column, - int[] bitmaps, - int columnIndex, - DataOutputStream dataOutputStream, - DataOutputStream dataBitmapOutputStream) - throws IOException { - for (int i = 0, size = column.getPositionCount(); i < size; i++) { - rowCount++; - if (column.isNull(i)) { - bitmaps[columnIndex] = bitmaps[columnIndex] << 1; - } else { - bitmaps[columnIndex] = (bitmaps[columnIndex] << 1) | FLAG; - dataOutputStream.writeInt(column.getInt(i)); - } - if (rowCount != 0 && rowCount % 8 == 0) { - dataBitmapOutputStream.writeByte(bitmaps[columnIndex]); - // we should clear the bitmap every 8 points - bitmaps[columnIndex] = 0; - } - } - } - - private static void doWithInt64Column( - int rowCount, - Column column, - int[] bitmaps, - int columnIndex, - DataOutputStream dataOutputStream, - DataOutputStream dataBitmapOutputStream) - throws IOException { - for (int i = 0, size = column.getPositionCount(); i < size; i++) { - rowCount++; - if (column.isNull(i)) { - bitmaps[columnIndex] = bitmaps[columnIndex] << 1; - } else { - bitmaps[columnIndex] = (bitmaps[columnIndex] << 1) | FLAG; - dataOutputStream.writeLong(column.getLong(i)); - } - if (rowCount != 0 && rowCount % 8 == 0) { - dataBitmapOutputStream.writeByte(bitmaps[columnIndex]); - // we should clear the bitmap every 8 points - bitmaps[columnIndex] = 0; - } - } - } - - private static void doWithFloatColumn( - int rowCount, - Column column, - int[] bitmaps, - int columnIndex, - DataOutputStream dataOutputStream, - DataOutputStream dataBitmapOutputStream) - throws IOException { - for (int i = 0, size = column.getPositionCount(); i < size; i++) { - rowCount++; - if (column.isNull(i)) { - bitmaps[columnIndex] = bitmaps[columnIndex] << 1; - } else { - bitmaps[columnIndex] = (bitmaps[columnIndex] << 1) | FLAG; - dataOutputStream.writeFloat(column.getFloat(i)); - } - if (rowCount != 0 && rowCount % 8 == 0) { - dataBitmapOutputStream.writeByte(bitmaps[columnIndex]); - // we should clear the bitmap every 8 points - bitmaps[columnIndex] = 0; - } - } - } - - private static void doWithDoubleColumn( - int rowCount, - Column column, - int[] bitmaps, - int columnIndex, - DataOutputStream dataOutputStream, - DataOutputStream dataBitmapOutputStream) - throws IOException { - for (int i = 0, size = column.getPositionCount(); i < size; i++) { - rowCount++; - if (column.isNull(i)) { - bitmaps[columnIndex] = bitmaps[columnIndex] << 1; - } else { - bitmaps[columnIndex] = (bitmaps[columnIndex] << 1) | FLAG; - dataOutputStream.writeDouble(column.getDouble(i)); - } - if (rowCount != 0 && rowCount % 8 == 0) { - dataBitmapOutputStream.writeByte(bitmaps[columnIndex]); - // we should clear the bitmap every 8 points - bitmaps[columnIndex] = 0; - } - } - } - - private static void doWithBooleanColumn( - int rowCount, - Column column, - int[] bitmaps, - int columnIndex, - DataOutputStream dataOutputStream, - DataOutputStream dataBitmapOutputStream) - throws IOException { - for (int i = 0, size = column.getPositionCount(); i < size; i++) { - rowCount++; - if (column.isNull(i)) { - bitmaps[columnIndex] = bitmaps[columnIndex] << 1; - } else { - bitmaps[columnIndex] = (bitmaps[columnIndex] << 1) | FLAG; - dataOutputStream.writeBoolean(column.getBoolean(i)); - } - if (rowCount != 0 && rowCount % 8 == 0) { - dataBitmapOutputStream.writeByte(bitmaps[columnIndex]); - // we should clear the bitmap every 8 points - bitmaps[columnIndex] = 0; - } + serializeColumn(rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); } } - private static void doWithTextColumn( + private static void serializeColumn( int rowCount, Column column, int[] bitmaps, @@ -468,9 +190,7 @@ private static void doWithTextColumn( bitmaps[columnIndex] = bitmaps[columnIndex] << 1; } else { bitmaps[columnIndex] = (bitmaps[columnIndex] << 1) | FLAG; - Binary binary = column.getBinary(i); - dataOutputStream.writeInt(binary.getLength()); - dataOutputStream.write(binary.getValues()); + column.writeTo(i, dataOutputStream); } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[columnIndex]); @@ -624,64 +344,10 @@ public static Object[] readTabletValuesFromBuffer( ByteBuffer buffer, TSDataType[] types, int columns, int size) { Object[] values = new Object[columns]; for (int i = 0; i < columns; i++) { - switch (types[i]) { - case BOOLEAN: - boolean[] boolValues = new boolean[size]; - for (int index = 0; index < size; index++) { - boolValues[index] = BytesUtils.byteToBool(buffer.get()); - } - values[i] = boolValues; - break; - case INT32: - case DATE: - int[] intValues = new int[size]; - for (int index = 0; index < size; index++) { - intValues[index] = buffer.getInt(); - } - values[i] = intValues; - break; - case INT64: - case TIMESTAMP: - long[] longValues = new long[size]; - for (int index = 0; index < size; index++) { - longValues[index] = buffer.getLong(); - } - values[i] = longValues; - break; - case FLOAT: - float[] floatValues = new float[size]; - for (int index = 0; index < size; index++) { - floatValues[index] = buffer.getFloat(); - } - values[i] = floatValues; - break; - case DOUBLE: - double[] doubleValues = new double[size]; - for (int index = 0; index < size; index++) { - doubleValues[index] = buffer.getDouble(); - } - values[i] = doubleValues; - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary[] binaryValues = new Binary[size]; - for (int index = 0; index < size; index++) { - int binarySize = buffer.getInt(); - byte[] binaryValue = new byte[binarySize]; - buffer.get(binaryValue); - binaryValues[index] = new Binary(binaryValue); - } - values[i] = binaryValues; - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages - .MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_WHEN_CONVERT_DATA_AT_CLIENT_405429CC, - types[i])); - } + values[i] = + TypeServices.StorageEngine.RAW_ARRAY_BYTE_BUFFER_DESERIALIZER_SERVICE + .call(Type.fromTsDataType(types[i])) + .apply(buffer, size); } return values; } @@ -690,95 +356,11 @@ public static Object[] readTabletValuesFromStream( DataInputStream stream, TSDataType[] types, int columns, int size) throws IOException { Object[] values = new Object[columns]; for (int i = 0; i < columns; i++) { - switch (types[i]) { - case BOOLEAN: - parseBooleanColumn(size, stream, values, i); - break; - case DATE: - case INT32: - parseInt32Column(size, stream, values, i); - break; - case TIMESTAMP: - case INT64: - parseInt64Column(size, stream, values, i); - break; - case FLOAT: - parseFloatColumn(size, stream, values, i); - break; - case DOUBLE: - parseDoubleColumn(size, stream, values, i); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - parseTextColumn(size, stream, values, i); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages - .MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_WHEN_CONVERT_DATA_AT_CLIENT_405429CC, - types[i])); - } + values[i] = + TypeServices.StorageEngine.RAW_ARRAY_INPUT_STREAM_DESERIALIZER_SERVICE + .call(Type.fromTsDataType(types[i])) + .deserialize(stream, size); } return values; } - - private static void parseBooleanColumn( - int size, DataInputStream stream, Object[] values, int columnIndex) throws IOException { - boolean[] boolValues = new boolean[size]; - for (int index = 0; index < size; index++) { - boolValues[index] = BytesUtils.byteToBool(stream.readByte()); - } - values[columnIndex] = boolValues; - } - - private static void parseInt32Column( - int size, DataInputStream stream, Object[] values, int columnIndex) throws IOException { - int[] intValues = new int[size]; - for (int index = 0; index < size; index++) { - intValues[index] = stream.readInt(); - } - values[columnIndex] = intValues; - } - - private static void parseInt64Column( - int size, DataInputStream stream, Object[] values, int columnIndex) throws IOException { - long[] longValues = new long[size]; - for (int index = 0; index < size; index++) { - longValues[index] = stream.readLong(); - } - values[columnIndex] = longValues; - } - - private static void parseFloatColumn( - int size, DataInputStream stream, Object[] values, int columnIndex) throws IOException { - float[] floatValues = new float[size]; - for (int index = 0; index < size; index++) { - floatValues[index] = stream.readFloat(); - } - values[columnIndex] = floatValues; - } - - private static void parseDoubleColumn( - int size, DataInputStream stream, Object[] values, int columnIndex) throws IOException { - double[] doubleValues = new double[size]; - for (int index = 0; index < size; index++) { - doubleValues[index] = stream.readDouble(); - } - values[columnIndex] = doubleValues; - } - - private static void parseTextColumn( - int size, DataInputStream stream, Object[] values, int columnIndex) throws IOException { - Binary[] binaryValues = new Binary[size]; - for (int index = 0; index < size; index++) { - int binarySize = stream.readInt(); - byte[] binaryValue = new byte[binarySize]; - stream.readFully(binaryValue); - binaryValues[index] = new Binary(binaryValue); - } - values[columnIndex] = binaryValues; - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 0ca603570734..8b1975a13542 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -37,13 +37,12 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.LongColumn; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -61,7 +60,6 @@ public class SchemaUtils { private static final Set canNotUseStatisticAfterAlterClassSet = ImmutableSet.of(TSDataType.STRING, TSDataType.TEXT, TSDataType.BLOB); public static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); - private static final Binary EMPTY_BINARY = new Binary("", StandardCharsets.UTF_8); static { dataTypeColumnClassMap = new HashMap<>(); @@ -466,164 +464,8 @@ public static TSEncoding getDataTypeCompatibleEncoding(TSDataType dataType, TSEn public static Statistics getNewStatistics( IChunkMetadata chunkMetadata, TSDataType targetDataType, Statistics statistics) { - switch (chunkMetadata.getDataType()) { - case INT32: - case INT64: - case TIMESTAMP: - case FLOAT: - case DOUBLE: - case BOOLEAN: - if (targetDataType == TSDataType.STRING) { - Binary[] binaryValues = new Binary[4]; - binaryValues[0] = - new Binary( - chunkMetadata.getStatistics().getFirstValue().toString(), StandardCharsets.UTF_8); - binaryValues[1] = - new Binary( - chunkMetadata.getStatistics().getLastValue().toString(), StandardCharsets.UTF_8); - if (chunkMetadata.getDataType() == TSDataType.BOOLEAN) { - binaryValues[2] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); - binaryValues[3] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); - } else { - binaryValues[2] = - new Binary( - chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8); - binaryValues[3] = - new Binary( - chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); - } - long[] longValues = new long[4]; - longValues[0] = chunkMetadata.getStatistics().getStartTime(); - longValues[1] = chunkMetadata.getStatistics().getEndTime(); - longValues[2] = longValues[1]; - longValues[3] = longValues[1]; - statistics.update(longValues, binaryValues, binaryValues.length); - } else if (targetDataType == TSDataType.TEXT) { - Binary[] binaryValues = new Binary[2]; - if (chunkMetadata.getDataType() == TSDataType.BOOLEAN) { - binaryValues[0] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); - binaryValues[1] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); - } else { - binaryValues[0] = - new Binary( - chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8); - binaryValues[1] = - new Binary( - chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); - } - long[] longValues = new long[2]; - longValues[0] = chunkMetadata.getStatistics().getStartTime(); - longValues[1] = chunkMetadata.getStatistics().getEndTime(); - statistics.update(longValues, binaryValues, binaryValues.length); - } else { - statistics = chunkMetadata.getStatistics(); - } - break; - case DATE: - if (targetDataType == TSDataType.STRING) { - Binary[] binaryValues = new Binary[4]; - binaryValues[0] = - new Binary( - TSDataType.getDateStringValue( - (Integer) chunkMetadata.getStatistics().getFirstValue()), - StandardCharsets.UTF_8); - binaryValues[1] = - new Binary( - TSDataType.getDateStringValue( - (Integer) chunkMetadata.getStatistics().getLastValue()), - StandardCharsets.UTF_8); - binaryValues[2] = - new Binary( - TSDataType.getDateStringValue( - (Integer) chunkMetadata.getStatistics().getMinValue()), - StandardCharsets.UTF_8); - binaryValues[3] = - new Binary( - TSDataType.getDateStringValue( - (Integer) chunkMetadata.getStatistics().getMaxValue()), - StandardCharsets.UTF_8); - long[] longValues = new long[4]; - longValues[0] = chunkMetadata.getStatistics().getStartTime(); - longValues[1] = chunkMetadata.getStatistics().getEndTime(); - longValues[2] = longValues[1]; - longValues[3] = longValues[1]; - statistics.update(longValues, binaryValues, binaryValues.length); - } else if (targetDataType == TSDataType.TEXT) { - Binary[] binaryValues = new Binary[2]; - binaryValues[0] = - new Binary( - TSDataType.getDateStringValue( - (Integer) chunkMetadata.getStatistics().getFirstValue()), - StandardCharsets.UTF_8); - binaryValues[1] = - new Binary( - TSDataType.getDateStringValue( - (Integer) chunkMetadata.getStatistics().getLastValue()), - StandardCharsets.UTF_8); - long[] longValues = new long[2]; - longValues[0] = chunkMetadata.getStatistics().getStartTime(); - longValues[1] = chunkMetadata.getStatistics().getEndTime(); - statistics.update(longValues, binaryValues, binaryValues.length); - } - break; - case STRING: - if (targetDataType == TSDataType.TEXT) { - Binary[] binaryValues = new Binary[2]; - binaryValues[0] = - new Binary( - chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8); - binaryValues[1] = - new Binary( - chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); - long[] longValues = new long[2]; - longValues[0] = chunkMetadata.getStatistics().getStartTime(); - longValues[1] = chunkMetadata.getStatistics().getEndTime(); - statistics.update(longValues, binaryValues, binaryValues.length); - } else if (targetDataType == TSDataType.BLOB) { - statistics.update( - chunkMetadata.getStatistics().getStartTime(), - new Binary( - chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8)); - statistics.update( - chunkMetadata.getStatistics().getEndTime(), - new Binary( - chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8)); - } else { - statistics = chunkMetadata.getStatistics(); - } - break; - case TEXT: - if (targetDataType == TSDataType.STRING) { - Binary[] binaryValues = new Binary[2]; - binaryValues[0] = (Binary) chunkMetadata.getStatistics().getFirstValue(); - binaryValues[1] = (Binary) chunkMetadata.getStatistics().getLastValue(); - long[] longValues = new long[2]; - longValues[0] = chunkMetadata.getStatistics().getStartTime(); - longValues[1] = chunkMetadata.getStatistics().getEndTime(); - statistics.update(longValues, binaryValues, binaryValues.length); - } else if (targetDataType == TSDataType.BLOB) { - statistics.update(chunkMetadata.getStatistics().getStartTime(), EMPTY_BINARY); - statistics.update(chunkMetadata.getStatistics().getEndTime(), EMPTY_BINARY); - } else { - statistics = chunkMetadata.getStatistics(); - } - break; - case BLOB: - if (targetDataType == TSDataType.STRING || targetDataType == TSDataType.TEXT) { - Binary[] binaryValues = new Binary[2]; - binaryValues[0] = EMPTY_BINARY; - binaryValues[1] = EMPTY_BINARY; - long[] longValues = new long[2]; - longValues[0] = chunkMetadata.getStatistics().getStartTime(); - longValues[1] = chunkMetadata.getStatistics().getEndTime(); - statistics.update(longValues, binaryValues, binaryValues.length); - } else { - statistics = chunkMetadata.getStatistics(); - } - break; - default: - break; - } - return statistics; + return TypeServices.StorageEngine.CHUNK_METADATA_STATISTICS_CONVERTER_SERVICE + .call(Type.fromTsDataType(chunkMetadata.getDataType())) + .convert(chunkMetadata, targetDataType, statistics); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TabletDecoder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TabletDecoder.java index 2f1d73b72313..1d577c489e6f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TabletDecoder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TabletDecoder.java @@ -28,9 +28,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.IOException; import java.nio.ByteBuffer; @@ -163,10 +162,8 @@ private static boolean allValueColumnsPlain( } private static boolean supportsPlainFastPath(TSDataType dataType) { - return switch (dataType) { - case BOOLEAN, DATE, INT32, TIMESTAMP, INT64, FLOAT, DOUBLE, STRING, BLOB, TEXT -> true; - case VECTOR, UNKNOWN, OBJECT -> false; - }; + return TypeServices.StorageEngine.TABLET_PLAIN_FAST_PATH_SERVICE.call( + Type.fromTsDataType(dataType)); } private Object decodeColumn(ByteBuffer uncompressed, int columnIndex) { @@ -174,73 +171,8 @@ private Object decodeColumn(ByteBuffer uncompressed, int columnIndex) { TSEncoding encoding = columnEncodings.get(columnIndex + 1); Decoder decoder = Decoder.getDecoderByType(encoding, dataType); - Object column = null; - switch (dataType) { - case DATE: - case INT32: - int[] intCol = new int[rowSize]; - if (encoding == TSEncoding.PLAIN) { - // PlainEncoder uses var int, which may cause compatibility problem - for (int j = 0; j < rowSize; j++) { - intCol[j] = ReadWriteIOUtils.readInt(uncompressed); - } - } else { - for (int j = 0; j < rowSize; j++) { - intCol[j] = decoder.readInt(uncompressed); - } - } - column = intCol; - break; - case INT64: - case TIMESTAMP: - long[] longCol = new long[rowSize]; - for (int j = 0; j < rowSize; j++) { - longCol[j] = decoder.readLong(uncompressed); - } - column = longCol; - break; - case FLOAT: - float[] floatCol = new float[rowSize]; - for (int j = 0; j < rowSize; j++) { - floatCol[j] = decoder.readFloat(uncompressed); - } - column = floatCol; - break; - case DOUBLE: - double[] doubleCol = new double[rowSize]; - for (int j = 0; j < rowSize; j++) { - doubleCol[j] = decoder.readDouble(uncompressed); - } - column = doubleCol; - break; - case BOOLEAN: - boolean[] boolCol = new boolean[rowSize]; - for (int j = 0; j < rowSize; j++) { - boolCol[j] = decoder.readBoolean(uncompressed); - } - column = boolCol; - break; - case STRING: - case BLOB: - case TEXT: - Binary[] binaryCol = new Binary[rowSize]; - if (encoding == TSEncoding.PLAIN) { - // PlainEncoder uses var int, which may cause compatibility problem - for (int j = 0; j < rowSize; j++) { - binaryCol[j] = ReadWriteIOUtils.readBinary(uncompressed); - } - } else { - for (int j = 0; j < rowSize; j++) { - binaryCol[j] = decoder.readBinary(uncompressed); - } - } - column = binaryCol; - break; - case UNKNOWN: - case VECTOR: - default: - throw new IllegalArgumentException(DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + dataType); - } - return column; + return TypeServices.StorageEngine.TABLET_COLUMN_DECODER_SERVICE + .call(Type.fromTsDataType(dataType)) + .decode(decoder, uncompressed, rowSize, encoding); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java index 6f895b7ec6a2..96cf8b16f66e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java @@ -18,19 +18,9 @@ */ package org.apache.iotdb.db.utils; -import org.apache.iotdb.db.i18n.DataNodeMiscMessages; - -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.TsPrimitiveType.TsBinary; -import org.apache.tsfile.utils.TsPrimitiveType.TsBoolean; -import org.apache.tsfile.utils.TsPrimitiveType.TsDouble; -import org.apache.tsfile.utils.TsPrimitiveType.TsFloat; -import org.apache.tsfile.utils.TsPrimitiveType.TsInt; -import org.apache.tsfile.utils.TsPrimitiveType.TsLong; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.read.common.type.Type; import java.util.ArrayList; @@ -46,58 +36,15 @@ private TimeValuePairUtils() {} */ public static void setTimeValuePair(TimeValuePair from, TimeValuePair to) { to.setTimestamp(from.getTimestamp()); - switch (from.getValue().getDataType()) { - case INT32: - case DATE: - to.getValue().setInt(from.getValue().getInt()); - break; - case INT64: - case TIMESTAMP: - to.getValue().setLong(from.getValue().getLong()); - break; - case FLOAT: - to.getValue().setFloat(from.getValue().getFloat()); - break; - case DOUBLE: - to.getValue().setDouble(from.getValue().getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - to.getValue().setBinary(from.getValue().getBinary()); - break; - case BOOLEAN: - to.getValue().setBoolean(from.getValue().getBoolean()); - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(from.getValue().getDataType())); - } + to.getValue().copy(from.getValue()); } public static TimeValuePair getEmptyTimeValuePair(TSDataType dataType) { - switch (dataType) { - case FLOAT: - return new TimeValuePair(0, new TsFloat(0.0f)); - case INT32: - case DATE: - return new TimeValuePair(0, new TsInt(0)); - case INT64: - case TIMESTAMP: - return new TimeValuePair(0, new TsLong(0)); - case BOOLEAN: - return new TimeValuePair(0, new TsBoolean(false)); - case DOUBLE: - return new TimeValuePair(0, new TsDouble(0.0)); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return new TimeValuePair(0, new TsBinary(new Binary("", TSFileConfig.STRING_CHARSET))); - default: - throw new UnsupportedOperationException( - DataNodeMiscMessages.UNRECOGNIZED_DATATYPE + dataType); - } + return new TimeValuePair( + 0, + TypeServices.StorageEngine.EMPTY_TS_PRIMITIVE_TYPE_FACTORY_SERVICE + .call(Type.fromTsDataType(dataType)) + .get()); } /** All intervals are closed. */ diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java index d48ed5fffd10..4f8424cfaacb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java @@ -35,10 +35,13 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.lang3.StringUtils; +import org.apache.tsfile.read.common.type.Type; import java.util.Collections; import java.util.List; +import static org.apache.iotdb.db.utils.TypeServices.ValueConversion.AUTO_CAST_SERVICE; + public class TypeInferenceUtils { private static final IoTDBConfig CONF = IoTDBDescriptor.getInstance().getConfig(); @@ -358,30 +361,10 @@ public static boolean canAutoCast(TSDataType fromType, TSDataType toType) { return true; } - switch (fromType) { - case INT32: - switch (toType) { - case INT64: - case FLOAT: - case DOUBLE: - return true; - default: - return false; - } - case INT64: - case FLOAT: - return toType.equals(TSDataType.DOUBLE); - case DOUBLE: - case BOOLEAN: - case TEXT: - case DATE: - case TIMESTAMP: - case BLOB: - case OBJECT: - case STRING: - return false; - default: - throw new IllegalArgumentException(DataNodeMiscMessages.UNKNOWN_DATA_TYPE + fromType); + try { + return AUTO_CAST_SERVICE.call(Type.fromTsDataType(fromType)).test(toType); + } catch (final UnsupportedOperationException ignored) { + throw new IllegalArgumentException(DataNodeMiscMessages.UNKNOWN_DATA_TYPE + fromType); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java new file mode 100644 index 000000000000..5a7ed3b2f7a7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java @@ -0,0 +1,4276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import org.apache.iotdb.calc.exception.QueryProcessException; +import org.apache.iotdb.calc.execution.aggregation.Accumulator; +import org.apache.iotdb.calc.execution.aggregation.BinaryModeAccumulator; +import org.apache.iotdb.calc.execution.aggregation.BooleanModeAccumulator; +import org.apache.iotdb.calc.execution.aggregation.DoubleModeAccumulator; +import org.apache.iotdb.calc.execution.aggregation.FloatModeAccumulator; +import org.apache.iotdb.calc.execution.aggregation.IntModeAccumulator; +import org.apache.iotdb.calc.execution.aggregation.LongModeAccumulator; +import org.apache.iotdb.calc.execution.operator.process.fill.IFill; +import org.apache.iotdb.calc.execution.operator.process.fill.constant.BinaryConstantFill; +import org.apache.iotdb.calc.execution.operator.process.fill.constant.BooleanConstantFill; +import org.apache.iotdb.calc.execution.operator.process.fill.constant.DoubleConstantFill; +import org.apache.iotdb.calc.execution.operator.process.fill.constant.FloatConstantFill; +import org.apache.iotdb.calc.execution.operator.process.fill.constant.IntConstantFill; +import org.apache.iotdb.calc.execution.operator.process.fill.constant.LongConstantFill; +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.calc.plan.planner.CommonOperatorUtils; +import org.apache.iotdb.calc.transformation.dag.util.CastFunctionUtils; +import org.apache.iotdb.calc.utils.TypeServices.PrimitiveColumnValueSetter; +import org.apache.iotdb.calc.utils.constant.SqlConstant; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BinaryLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FloatLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.GenericLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Literal; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; +import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.i18n.DataNodeMiscMessages; +import org.apache.iotdb.db.i18n.DataNodePipeMessages; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.pipe.processor.aggregate.TimeSeriesRuntimeState; +import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.sametype.numeric.AbstractSameTypeNumericOperator; +import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowOutput; +import org.apache.iotdb.db.queryengine.execution.operator.window.EqualBinaryWindowManager; +import org.apache.iotdb.db.queryengine.execution.operator.window.EqualBooleanWindowManager; +import org.apache.iotdb.db.queryengine.execution.operator.window.EqualDoubleWindowManager; +import org.apache.iotdb.db.queryengine.execution.operator.window.EqualFloatWindowManager; +import org.apache.iotdb.db.queryengine.execution.operator.window.EqualIntWindowManager; +import org.apache.iotdb.db.queryengine.execution.operator.window.EqualLongWindowManager; +import org.apache.iotdb.db.queryengine.execution.operator.window.VariationDoubleWindowManager; +import org.apache.iotdb.db.queryengine.execution.operator.window.VariationFloatWindowManager; +import org.apache.iotdb.db.queryengine.execution.operator.window.VariationIntWindowManager; +import org.apache.iotdb.db.queryengine.execution.operator.window.VariationLongWindowManager; +import org.apache.iotdb.db.queryengine.execution.operator.window.VariationWindowManager; +import org.apache.iotdb.db.queryengine.execution.operator.window.VariationWindowParameter; +import org.apache.iotdb.db.queryengine.statistics.StatisticsManager; +import org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.ArithmeticNegationTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.InTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.scalar.DiffFunctionTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.scalar.RoundFunctionTransformer; +import org.apache.iotdb.db.queryengine.transformation.datastructure.util.ValueRecorder; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; +import org.apache.iotdb.db.utils.datastructure.AlignedTVList; +import org.apache.iotdb.db.utils.datastructure.BinaryTVList; +import org.apache.iotdb.db.utils.datastructure.BooleanTVList; +import org.apache.iotdb.db.utils.datastructure.DoubleTVList; +import org.apache.iotdb.db.utils.datastructure.FloatTVList; +import org.apache.iotdb.db.utils.datastructure.IntTVList; +import org.apache.iotdb.db.utils.datastructure.LongTVList; +import org.apache.iotdb.db.utils.datastructure.TVList; +import org.apache.iotdb.db.utils.windowing.window.EvictableBatchList; +import org.apache.iotdb.db.utils.windowing.window.WindowImpl; +import org.apache.iotdb.pipe.api.access.Row; +import org.apache.iotdb.rpc.RpcUtils; + +import com.google.common.io.BaseEncoding; +import com.sun.jna.platform.win32.OaIdl; +import com.sun.jna.platform.win32.OleAuto; +import com.sun.jna.platform.win32.Variant; +import com.sun.jna.platform.win32.WTypes; +import com.sun.jna.platform.win32.WinDef; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.external.commons.lang3.StringUtils; +import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.DateStatistics; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.read.filter.basic.Filter; +import org.apache.tsfile.read.reader.series.PaginationController; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.eclipse.milo.opcua.stack.core.Identifiers; +import org.eclipse.milo.opcua.stack.core.types.builtin.DateTime; +import org.eclipse.milo.opcua.stack.core.types.builtin.NodeId; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.sql.Date; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.IntFunction; +import java.util.function.LongSupplier; +import java.util.function.Supplier; +import java.util.function.ToDoubleFunction; +import java.util.function.ToIntFunction; +import java.util.function.ToLongFunction; + +public class TypeServices { + + public static final int DEFAULT_DATE = + DateUtils.parseDateExpressionToInt(LocalDate.of(1970, 1, 1)); + private static final LocalDate EMPTY_LOCAL_DATE = LocalDate.of(1000, 1, 1); + + public static final class Memory { + + public static final TypeService STATEMENT_VALUE_SIZE_PER_LINE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, DATE, INT64, TIMESTAMP, FLOAT, DOUBLE -> type::estimateValueSize; + case TEXT, BLOB, STRING -> + () -> StatisticsManager.getInstance().getMaxBinarySizeInBytes(); + case OBJECT, ROW, UNKNOWN, VECTOR -> + () -> { + throw new UnsupportedOperationException( + CalcMessages.UNKNOWN_DATATYPE + type.getTypeEnum()); + }; + }; + + public static final TypeService> INSERT_NODE_VALUE_SIZE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, DATE, INT64, TIMESTAMP, FLOAT, DOUBLE -> + value -> + RamUsageEstimator.alignObjectSize( + type.estimateValueSize() + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER); + case TEXT, BLOB, STRING -> + value -> Objects.nonNull(value) ? ((Binary) value).ramBytesUsed() : 0L; + case OBJECT, ROW, UNKNOWN, VECTOR -> value -> 0L; + }; + + static { + STATEMENT_VALUE_SIZE_PER_LINE_SERVICE.check(); + INSERT_NODE_VALUE_SIZE_SERVICE.check(); + } + + private Memory() { + // Utility class + } + } + + public static final class Transformation { + + public static final TypeService> CONSTANT_COLUMN_BUILDER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT -> + value -> { + final ColumnBuilder builder = type.createColumnBuilder(1); + type.writeObject(builder, value); + return builder.build(); + }; + case DATE, TIMESTAMP, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_TYPE + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService< + Function> + CONSTANT_FILL_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> literal -> new BooleanConstantFill(literal.getBoolean()); + case INT32 -> literal -> new IntConstantFill(literal.getInt()); + case DATE -> literal -> new IntConstantFill(literal.getDate()); + case INT64, TIMESTAMP -> literal -> new LongConstantFill(literal.getLong()); + case FLOAT -> literal -> new FloatConstantFill(literal.getFloat()); + case DOUBLE -> literal -> new DoubleConstantFill(literal.getDouble()); + case TEXT, BLOB, STRING -> literal -> new BinaryConstantFill(literal.getBinary()); + case OBJECT, ROW, UNKNOWN, VECTOR -> + literal -> { + throw new IllegalArgumentException( + CommonOperatorUtils.UNKNOWN_DATATYPE + type.getTypeEnum()); + }; + }; + + public static final TypeService> COLUMN_BUILDER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, + INT32, + DATE, + INT64, + TIMESTAMP, + FLOAT, + DOUBLE, + TEXT, + BLOB, + STRING, + OBJECT -> + type::createColumnBuilder; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, + type.getTypeEnum())) + .setChecked(true); + }; + + public static final TypeService VALUE_TO_DOUBLE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> type::getInt; + case INT64, TIMESTAMP -> type::getLong; + case FLOAT -> type::getFloat; + case DOUBLE -> type::getDouble; + case BOOLEAN -> (column, index) -> type.getBoolean(column, index) ? 1 : 0; + case TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + (column, index) -> { + throw new QueryProcessException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()); + }; + }; + + public static final TypeService TRANSFORM_COLUMN_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, + INT32, + DATE, + INT64, + TIMESTAMP, + FLOAT, + DOUBLE, + TEXT, + BLOB, + STRING, + OBJECT -> + type::write; + case ROW, UNKNOWN, VECTOR -> + (builder, column, index) -> { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, + type.getTypeEnum())); + }; + }; + + // Schema evolution keeps physically compatible columns, widens supported values, and exposes + // incompatible historical values as nulls instead of interpreting their raw storage. + public static final TypeService + ALTERED_DATA_TYPE_COLUMN_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, + INT32, + INT64, + FLOAT, + DOUBLE, + TEXT, + STRING, + TIMESTAMP, + DATE, + BLOB -> { + final TSDataType targetType = TSDataType.valueOf(type.getTypeEnum().name()); + yield (source, positionCount) -> { + final TSDataType sourceType = source.getDataType(); + // DATE shares INT32's physical column but was historically incompatible with + // a schema change to INT32, while the other shared-column pairs are reusable. + if (sourceType == targetType + || (targetType != TSDataType.INT32 + && SchemaUtils.isUsingSameColumn(sourceType, targetType))) { + return source; + } + return targetType.isCompatible(sourceType) + ? source.convertTo(targetType) + : type.createNullColumn(positionCount); + }; + } + case OBJECT -> (source, positionCount) -> source; + case ROW, UNKNOWN, VECTOR -> (source, positionCount) -> null; + }; + + public static final TypeService ROUND_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> RoundFunctionTransformer::transformInt; + case INT64 -> RoundFunctionTransformer::transformLong; + case FLOAT -> RoundFunctionTransformer::transformFloat; + case DOUBLE -> RoundFunctionTransformer::transformDouble; + case BOOLEAN, DATE, TIMESTAMP, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + (transformer, columns, builder) -> { + throw new UnsupportedOperationException( + String.format("Unsupported source dataType: %s", type.getTypeEnum())); + }; + }; + + public static final TypeService DIFF_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> DiffFunctionTransformer::transformInt; + case INT64 -> DiffFunctionTransformer::transformLong; + case FLOAT -> DiffFunctionTransformer::transformFloat; + case DOUBLE -> DiffFunctionTransformer::transformDouble; + case BOOLEAN, DATE, TIMESTAMP, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + (transformer, columns, builder) -> { + throw new QueryProcessException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()); + }; + }; + + public static final TypeService NEGATION_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> ArithmeticNegationTransformer::transformInt; + case INT64 -> ArithmeticNegationTransformer::transformLong; + case FLOAT -> ArithmeticNegationTransformer::transformFloat; + case DOUBLE -> ArithmeticNegationTransformer::transformDouble; + case BOOLEAN, DATE, TIMESTAMP, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + (transformer, columns, builder) -> { + throw new QueryProcessException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()); + }; + }; + + public static final TypeService STATE_WINDOW_SPLITTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (valueRecorder, delta, values, index) -> { + if (!valueRecorder.hasRecorded()) { + valueRecorder.recordInt(type.getInt(values, index - 1)); + valueRecorder.setRecorded(true); + } + final boolean split = + Math.abs(type.getInt(values, index) - valueRecorder.getInt()) > delta; + if (split) { + valueRecorder.recordInt(type.getInt(values, index)); + } + return split; + }; + case INT64 -> + (valueRecorder, delta, values, index) -> { + if (!valueRecorder.hasRecorded()) { + valueRecorder.recordLong(type.getLong(values, index - 1)); + valueRecorder.setRecorded(true); + } + final boolean split = + Math.abs(type.getLong(values, index) - valueRecorder.getLong()) > delta; + if (split) { + valueRecorder.recordLong(type.getLong(values, index)); + } + return split; + }; + case FLOAT -> + (valueRecorder, delta, values, index) -> { + if (!valueRecorder.hasRecorded()) { + valueRecorder.recordFloat(type.getFloat(values, index - 1)); + valueRecorder.setRecorded(true); + } + final boolean split = + Math.abs(type.getFloat(values, index) - valueRecorder.getFloat()) > delta; + if (split) { + valueRecorder.recordFloat(type.getFloat(values, index)); + } + return split; + }; + case DOUBLE -> + (valueRecorder, delta, values, index) -> { + if (!valueRecorder.hasRecorded()) { + valueRecorder.recordDouble(type.getDouble(values, index - 1)); + valueRecorder.setRecorded(true); + } + final boolean split = + Math.abs(type.getDouble(values, index) - valueRecorder.getDouble()) > delta; + if (split) { + valueRecorder.recordDouble(type.getDouble(values, index)); + } + return split; + }; + case BOOLEAN -> + (valueRecorder, delta, values, index) -> { + if (!valueRecorder.hasRecorded()) { + valueRecorder.recordBoolean(type.getBoolean(values, index - 1)); + valueRecorder.setRecorded(true); + } + final boolean split = + type.getBoolean(values, index) != valueRecorder.getBoolean(); + if (split) { + valueRecorder.recordBoolean(type.getBoolean(values, index)); + } + return split; + }; + case TEXT -> + (valueRecorder, delta, values, index) -> { + if (!valueRecorder.hasRecorded()) { + valueRecorder.recordString(type.getBinary(values, index - 1).toString()); + valueRecorder.setRecorded(true); + } + final String value = type.getBinary(values, index).toString(); + final boolean split = !value.equals(valueRecorder.getString()); + if (split) { + valueRecorder.recordString(value); + } + return split; + }; + case DATE, TIMESTAMP, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + (valueRecorder, delta, values, index) -> { + throw new UnsupportedOperationException( + DataNodeQueryMessages + .EXCEPTION_THE_DATA_TYPE_OF_THE_STATE_WINDOW_STRATEGY_IS_NOT_VALID_61F4E273); + }; + }; + + public static final TypeService + IN_TRANSFORMER_SET_INITIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> InTransformer::initIntSet; + case INT64, TIMESTAMP -> InTransformer::initLongSet; + case FLOAT -> InTransformer::initFloatSet; + case DOUBLE -> InTransformer::initDoubleSet; + case BOOLEAN -> InTransformer::initBooleanSet; + case TEXT, STRING -> InTransformer::initStringSet; + case BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (transformer, values) -> { + throw new UnsupportedOperationException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_3 + type.getTypeEnum()); + }; + }; + + public static final TypeService + IN_TRANSFORMER_COLUMN_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> InTransformer::transformInt; + case INT64, TIMESTAMP -> InTransformer::transformLong; + case FLOAT -> InTransformer::transformFloat; + case DOUBLE -> InTransformer::transformDouble; + case BOOLEAN -> InTransformer::transformBoolean; + case TEXT, STRING -> InTransformer::transformBinary; + case BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (transformer, columns, builder) -> { + throw new QueryProcessException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_3 + type.getTypeEnum()); + }; + }; + + public static final TypeService> CAST_COLUMN_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + targetType -> + switch (targetType.getTypeEnum()) { + case INT32 -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeInt(builder, type.getInt(column, index))); + case INT64 -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeLong(builder, type.getInt(column, index))); + case FLOAT -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeFloat(builder, type.getInt(column, index))); + case DOUBLE -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeDouble(builder, type.getInt(column, index))); + case BOOLEAN -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeBoolean( + builder, type.getInt(column, index) != 0)); + case TEXT -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeBinary( + builder, + BytesUtils.valueOf( + String.valueOf(type.getInt(column, index))))); + case DATE, TIMESTAMP, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + unsupportedTargetCastStrategy(type); + }; + case INT64 -> + targetType -> + switch (targetType.getTypeEnum()) { + case INT32 -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeInt( + builder, + CastFunctionUtils.castLongToInt( + type.getLong(column, index)))); + case INT64 -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeLong(builder, type.getLong(column, index))); + case FLOAT -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeFloat(builder, type.getLong(column, index))); + case DOUBLE -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeDouble(builder, type.getLong(column, index))); + case BOOLEAN -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeBoolean( + builder, type.getLong(column, index) != 0L)); + case TEXT -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeBinary( + builder, + BytesUtils.valueOf( + String.valueOf(type.getLong(column, index))))); + case DATE, TIMESTAMP, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + unsupportedTargetCastStrategy(type); + }; + case FLOAT -> + targetType -> + switch (targetType.getTypeEnum()) { + case INT32 -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeInt( + builder, + CastFunctionUtils.castFloatToInt( + type.getFloat(column, index)))); + case INT64 -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeLong( + builder, + CastFunctionUtils.castFloatToLong( + type.getFloat(column, index)))); + case FLOAT -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeFloat(builder, type.getFloat(column, index))); + case DOUBLE -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeDouble(builder, type.getFloat(column, index))); + case BOOLEAN -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeBoolean( + builder, type.getFloat(column, index) != 0.0f)); + case TEXT -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeBinary( + builder, + BytesUtils.valueOf( + String.valueOf(type.getFloat(column, index))))); + case DATE, TIMESTAMP, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + unsupportedTargetCastStrategy(type); + }; + case DOUBLE -> + targetType -> + switch (targetType.getTypeEnum()) { + case INT32 -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeInt( + builder, + CastFunctionUtils.castDoubleToInt( + type.getDouble(column, index)))); + case INT64 -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeLong( + builder, + CastFunctionUtils.castDoubleToLong( + type.getDouble(column, index)))); + case FLOAT -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeFloat( + builder, + CastFunctionUtils.castDoubleToFloat( + type.getDouble(column, index)))); + case DOUBLE -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeDouble(builder, type.getDouble(column, index))); + case BOOLEAN -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeBoolean( + builder, type.getDouble(column, index) != 0.0)); + case TEXT -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeBinary( + builder, + BytesUtils.valueOf( + String.valueOf(type.getDouble(column, index))))); + case DATE, TIMESTAMP, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + unsupportedTargetCastStrategy(type); + }; + case BOOLEAN -> + targetType -> + switch (targetType.getTypeEnum()) { + case INT32 -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeInt( + builder, type.getBoolean(column, index) ? 1 : 0)); + case INT64 -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeLong( + builder, type.getBoolean(column, index) ? 1L : 0L)); + case FLOAT -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeFloat( + builder, type.getBoolean(column, index) ? 1.0f : 0.0f)); + case DOUBLE -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeDouble( + builder, type.getBoolean(column, index) ? 1.0 : 0.0)); + case BOOLEAN -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeBoolean( + builder, type.getBoolean(column, index))); + case TEXT -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeBinary( + builder, + BytesUtils.valueOf( + String.valueOf(type.getBoolean(column, index))))); + case DATE, TIMESTAMP, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + unsupportedTargetCastStrategy(type); + }; + case TEXT -> + targetType -> + switch (targetType.getTypeEnum()) { + case INT32 -> + castBinaryColumnStrategy( + type, + targetType, + (value, builder) -> + targetType.writeInt(builder, Integer.parseInt(value))); + case INT64 -> + castBinaryColumnStrategy( + type, + targetType, + (value, builder) -> + targetType.writeLong(builder, Long.parseLong(value))); + case FLOAT -> + castBinaryColumnStrategy( + type, + targetType, + (value, builder) -> + targetType.writeFloat( + builder, CastFunctionUtils.castTextToFloat(value))); + case DOUBLE -> + castBinaryColumnStrategy( + type, + targetType, + (value, builder) -> + targetType.writeDouble( + builder, CastFunctionUtils.castTextToDouble(value))); + case BOOLEAN -> + castBinaryColumnStrategy( + type, + targetType, + (value, builder) -> + targetType.writeBoolean( + builder, CastFunctionUtils.castTextToBoolean(value))); + case TEXT -> + castColumnStrategy( + targetType, + (column, index, builder) -> + targetType.writeBinary(builder, type.getBinary(column, index))); + case DATE, TIMESTAMP, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + unsupportedTargetCastStrategy(type); + }; + case DATE, TIMESTAMP, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + targetType -> unsupportedSourceCastStrategy(type); + }; + + private static CastColumnStrategy castColumnStrategy( + final Type targetType, final CastColumnWriter writer) { + return new CastColumnStrategy(() -> {}, targetType::createColumnBuilder, writer); + } + + private static CastColumnStrategy castBinaryColumnStrategy( + final Type sourceType, final Type targetType, final BinaryStringCastWriter writer) { + return castColumnStrategy( + targetType, + (column, index, builder) -> + writer.write( + sourceType.getBinary(column, index).getStringValue(TSFileConfig.STRING_CHARSET), + builder)); + } + + private static CastColumnStrategy unsupportedSourceCastStrategy(final Type sourceType) { + return unsupportedCastStrategy( + String.format("Unsupported source dataType: %s", sourceType.getTypeEnum())); + } + + private static CastColumnStrategy unsupportedTargetCastStrategy(final Type sourceType) { + return unsupportedCastStrategy( + String.format("Unsupported target dataType: %s", sourceType.getTypeEnum())); + } + + private static CastColumnStrategy unsupportedCastStrategy(final String message) { + return new CastColumnStrategy( + () -> { + throw new UnsupportedOperationException(message); + }, + ignored -> { + throw new UnsupportedOperationException(message); + }, + (column, index, builder) -> { + throw new UnsupportedOperationException(message); + }); + } + + public static final class CastColumnStrategy { + private final Runnable validator; + private final IntFunction builderFactory; + private final CastColumnWriter writer; + + private CastColumnStrategy( + final Runnable validator, + final IntFunction builderFactory, + final CastColumnWriter writer) { + this.validator = validator; + this.builderFactory = builderFactory; + this.writer = writer; + } + + public void validate() { + validator.run(); + } + + public ColumnBuilder createBuilder(final int expectedEntries) { + return builderFactory.apply(expectedEntries); + } + + public void cast(final Column column, final int index, final ColumnBuilder builder) { + writer.write(column, index, builder); + } + } + + @FunctionalInterface + private interface CastColumnWriter { + void write(Column column, int index, ColumnBuilder builder); + } + + @FunctionalInterface + private interface BinaryStringCastWriter { + void write(String value, ColumnBuilder builder); + } + + @FunctionalInterface + public interface ColumnToDoubleConverter { + double convert(Column column, int index) throws QueryProcessException; + } + + @FunctionalInterface + public interface ColumnValueWriter { + void write(ColumnBuilder builder, Column column, int index); + } + + @FunctionalInterface + public interface AlteredDataTypeColumnTransformer { + Column transform(Column source, int positionCount); + } + + @FunctionalInterface + public interface RoundTransformer { + void transform(RoundFunctionTransformer transformer, Column[] columns, ColumnBuilder builder) + throws QueryProcessException, IOException; + } + + @FunctionalInterface + public interface DiffTransformer { + void transform(DiffFunctionTransformer transformer, Column[] columns, ColumnBuilder builder) + throws QueryProcessException; + } + + @FunctionalInterface + public interface NegationTransformer { + void transform( + ArithmeticNegationTransformer transformer, Column[] columns, ColumnBuilder builder) + throws QueryProcessException, IOException; + } + + @FunctionalInterface + public interface StateWindowSplitter { + boolean split(ValueRecorder valueRecorder, double delta, Column values, int index); + } + + @FunctionalInterface + public interface InTransformerSetInitializer { + void initialize(InTransformer transformer, Set values); + } + + @FunctionalInterface + public interface InTransformerColumnTransformer { + void transform(InTransformer transformer, Column[] columns, ColumnBuilder builder) + throws QueryProcessException; + } + + static { + CONSTANT_COLUMN_BUILDER_SERVICE.check(); + CONSTANT_FILL_SERVICE.check(); + COLUMN_BUILDER_SERVICE.check(); + VALUE_TO_DOUBLE_SERVICE.check(); + TRANSFORM_COLUMN_VALUE_WRITER_SERVICE.check(); + ALTERED_DATA_TYPE_COLUMN_TRANSFORMER_SERVICE.check(); + ROUND_TRANSFORMER_SERVICE.check(); + DIFF_TRANSFORMER_SERVICE.check(); + NEGATION_TRANSFORMER_SERVICE.check(); + STATE_WINDOW_SPLITTER_SERVICE.check(); + IN_TRANSFORMER_SET_INITIALIZER_SERVICE.check(); + IN_TRANSFORMER_COLUMN_TRANSFORMER_SERVICE.check(); + CAST_COLUMN_SERVICE.check(); + } + + private Transformation() { + // Utility class + } + } + + public static final class Aggregation { + + public static final TypeService + EXTREME_VALUE_ACCUMULATOR_STRATEGY_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + extremeValueAccumulatorStrategy( + (accumulator, column, index) -> + accumulator.updateIntResult(column.getInt(index)), + (accumulator, value) -> + accumulator.updateIntResult(((Number) value).intValue()), + (accumulator, column) -> accumulator.getResult().setInt(column.getInt(0)), + (builder, result) -> builder.writeInt(result.getInt())); + case INT64, TIMESTAMP -> + extremeValueAccumulatorStrategy( + (accumulator, column, index) -> + accumulator.updateLongResult(column.getLong(index)), + (accumulator, value) -> + accumulator.updateLongResult(((Number) value).longValue()), + (accumulator, column) -> + accumulator.getResult().setLong(column.getLong(0)), + (builder, result) -> builder.writeLong(result.getLong())); + case FLOAT -> + extremeValueAccumulatorStrategy( + (accumulator, column, index) -> + accumulator.updateFloatResult(column.getFloat(index)), + (accumulator, value) -> + accumulator.updateFloatResult(((Number) value).floatValue()), + (accumulator, column) -> + accumulator.getResult().setFloat(column.getFloat(0)), + (builder, result) -> builder.writeFloat(result.getFloat())); + case DOUBLE -> + extremeValueAccumulatorStrategy( + (accumulator, column, index) -> + accumulator.updateDoubleResult(column.getDouble(index)), + (accumulator, value) -> + accumulator.updateDoubleResult(((Number) value).doubleValue()), + (accumulator, column) -> + accumulator.getResult().setDouble(column.getDouble(0)), + (builder, result) -> builder.writeDouble(result.getDouble())); + case STRING -> + extremeValueAccumulatorStrategy( + (accumulator, column, index) -> + accumulator.updateBinaryResult(column.getBinary(index)), + (accumulator, value) -> accumulator.updateBinaryResult((Binary) value), + (accumulator, column) -> + accumulator.getResult().setBinary(column.getBinary(0)), + (builder, result) -> builder.writeBinary(result.getBinary())); + case BOOLEAN, TEXT, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + ExtremeValueAccumulatorStrategy.unsupported(); + }; + + public static final TypeService + TIME_VALUE_ACCUMULATOR_STRATEGY_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + timeValueAccumulatorStrategy( + (accumulator, column, index, time) -> + accumulator.updateIntResult(column.getInt(index), time), + (accumulator, statistics, first) -> + accumulator.updateIntResult( + ((Number) getStatisticsValue(statistics, first)).intValue(), + getStatisticsTime(statistics, first)), + (accumulator, column) -> + accumulator.getTimeValueResult().setInt(column.getInt(0)), + (builder, result) -> builder.writeInt(result.getInt())); + case INT64, TIMESTAMP -> + timeValueAccumulatorStrategy( + (accumulator, column, index, time) -> + accumulator.updateLongResult(column.getLong(index), time), + (accumulator, statistics, first) -> + accumulator.updateLongResult( + ((Number) getStatisticsValue(statistics, first)).longValue(), + getStatisticsTime(statistics, first)), + (accumulator, column) -> + accumulator.getTimeValueResult().setLong(column.getLong(0)), + (builder, result) -> builder.writeLong(result.getLong())); + case FLOAT -> + timeValueAccumulatorStrategy( + (accumulator, column, index, time) -> + accumulator.updateFloatResult(column.getFloat(index), time), + (accumulator, statistics, first) -> + accumulator.updateFloatResult( + ((Number) getStatisticsValue(statistics, first)).floatValue(), + getStatisticsTime(statistics, first)), + (accumulator, column) -> + accumulator.getTimeValueResult().setFloat(column.getFloat(0)), + (builder, result) -> builder.writeFloat(result.getFloat())); + case DOUBLE -> + timeValueAccumulatorStrategy( + (accumulator, column, index, time) -> + accumulator.updateDoubleResult(column.getDouble(index), time), + (accumulator, statistics, first) -> + accumulator.updateDoubleResult( + ((Number) getStatisticsValue(statistics, first)).doubleValue(), + getStatisticsTime(statistics, first)), + (accumulator, column) -> + accumulator.getTimeValueResult().setDouble(column.getDouble(0)), + (builder, result) -> builder.writeDouble(result.getDouble())); + case TEXT, BLOB, STRING, OBJECT -> + timeValueAccumulatorStrategy( + (accumulator, column, index, time) -> + accumulator.updateBinaryResult(column.getBinary(index), time), + (accumulator, statistics, first) -> + accumulator.updateBinaryResult( + getBinaryStatisticsValue(statistics, first), + getStatisticsTime(statistics, first)), + (accumulator, column) -> + accumulator.getTimeValueResult().setBinary(column.getBinary(0)), + (builder, result) -> builder.writeBinary(result.getBinary())); + case BOOLEAN -> + timeValueAccumulatorStrategy( + (accumulator, column, index, time) -> + accumulator.updateBooleanResult(column.getBoolean(index), time), + (accumulator, statistics, first) -> + accumulator.updateBooleanResult( + (boolean) getStatisticsValue(statistics, first), + getStatisticsTime(statistics, first)), + (accumulator, column) -> + accumulator.getTimeValueResult().setBoolean(column.getBoolean(0)), + (builder, result) -> builder.writeBoolean(result.getBoolean())); + case ROW, UNKNOWN, VECTOR -> TimeValueAccumulatorStrategy.unsupported(); + }; + + public static final TypeService + MAX_MIN_BY_ACCUMULATOR_STRATEGY_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + maxMinByAccumulatorStrategy( + type, + (accumulator, time, yColumn, yIndex, xColumn, xIndex) -> + accumulator.updateIntResult( + time, yColumn.getInt(yIndex), xColumn, xIndex), + (accumulator, time, bytes, offset) -> + accumulator.updateIntResult( + time, + BytesUtils.bytesToInt(bytes, offset), + accumulator.readXFromBytesIntermediateInput( + bytes, offset + Integer.BYTES), + 0)); + case INT64, TIMESTAMP -> + maxMinByAccumulatorStrategy( + type, + (accumulator, time, yColumn, yIndex, xColumn, xIndex) -> + accumulator.updateLongResult( + time, yColumn.getLong(yIndex), xColumn, xIndex), + (accumulator, time, bytes, offset) -> + accumulator.updateLongResult( + time, + BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset), + accumulator.readXFromBytesIntermediateInput( + bytes, offset + Long.BYTES), + 0)); + case FLOAT -> + maxMinByAccumulatorStrategy( + type, + (accumulator, time, yColumn, yIndex, xColumn, xIndex) -> + accumulator.updateFloatResult( + time, yColumn.getFloat(yIndex), xColumn, xIndex), + (accumulator, time, bytes, offset) -> + accumulator.updateFloatResult( + time, + BytesUtils.bytesToFloat(bytes, offset), + accumulator.readXFromBytesIntermediateInput( + bytes, offset + Float.BYTES), + 0)); + case DOUBLE -> + maxMinByAccumulatorStrategy( + type, + (accumulator, time, yColumn, yIndex, xColumn, xIndex) -> + accumulator.updateDoubleResult( + time, yColumn.getDouble(yIndex), xColumn, xIndex), + (accumulator, time, bytes, offset) -> + accumulator.updateDoubleResult( + time, + BytesUtils.bytesToDouble(bytes, offset), + accumulator.readXFromBytesIntermediateInput( + bytes, offset + Double.BYTES), + 0)); + case STRING -> + maxMinByAccumulatorStrategy( + type, + (accumulator, time, yColumn, yIndex, xColumn, xIndex) -> + accumulator.updateBinaryResult( + time, yColumn.getBinary(yIndex), xColumn, xIndex), + TypeServices.Aggregation::updateBinaryIntermediate); + case TEXT, BLOB, OBJECT -> maxMinByAccumulatorStrategy(type, null, null); + case BOOLEAN -> maxMinByAccumulatorStrategy(type, null, null); + case ROW, UNKNOWN, VECTOR -> MaxMinByAccumulatorStrategy.unsupported(type); + }; + + public static final TypeService + EVENT_WINDOW_MANAGER_PROVIDER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + eventWindowManagerProvider( + EqualIntWindowManager::new, VariationIntWindowManager::new); + case INT64 -> + eventWindowManagerProvider( + EqualLongWindowManager::new, VariationLongWindowManager::new); + case FLOAT -> + eventWindowManagerProvider( + EqualFloatWindowManager::new, VariationFloatWindowManager::new); + case DOUBLE -> + eventWindowManagerProvider( + EqualDoubleWindowManager::new, VariationDoubleWindowManager::new); + case TEXT -> eventWindowManagerProvider(EqualBinaryWindowManager::new, null); + case BOOLEAN -> eventWindowManagerProvider(EqualBooleanWindowManager::new, null); + case DATE, TIMESTAMP, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + EventWindowManagerProvider.unsupported(); + }; + + private static EventWindowManagerProvider eventWindowManagerProvider( + final EventWindowManagerFactory equalFactory, + final EventWindowManagerFactory variationFactory) { + return new EventWindowManagerProvider(equalFactory, variationFactory); + } + + private static void updateBinaryIntermediate( + final MaxMinByAccumulator accumulator, + final long time, + final byte[] bytes, + final int offset) { + final int length = BytesUtils.bytesToInt(bytes, offset); + final int valueOffset = offset + Integer.BYTES; + accumulator.updateBinaryResult( + time, + new Binary(BytesUtils.subBytes(bytes, valueOffset, length)), + accumulator.readXFromBytesIntermediateInput(bytes, valueOffset + length), + 0); + } + + private static MaxMinByAccumulatorStrategy maxMinByAccumulatorStrategy( + final Type type, + final MaxMinByYColumnUpdater yColumnUpdater, + final MaxMinByYBytesUpdater yBytesUpdater) { + return new MaxMinByAccumulatorStrategy(type, true, yColumnUpdater, yBytesUpdater); + } + + private static TimeValueAccumulatorStrategy timeValueAccumulatorStrategy( + final TimeValueColumnUpdater columnUpdater, + final TimeValueStatisticsUpdater statisticsUpdater, + final TimeValueFinalSetter finalSetter, + final BiConsumer resultWriter) { + return new TimeValueAccumulatorStrategy( + columnUpdater, statisticsUpdater, finalSetter, resultWriter); + } + + private static Object getStatisticsValue(final Statistics statistics, final boolean first) { + return first ? statistics.getFirstValue() : statistics.getLastValue(); + } + + private static long getStatisticsTime(final Statistics statistics, final boolean first) { + return first ? statistics.getStartTime() : statistics.getEndTime(); + } + + private static Binary getBinaryStatisticsValue( + final Statistics statistics, final boolean first) { + final Object value = getStatisticsValue(statistics, first); + if (statistics instanceof DateStatistics) { + return new Binary(TSDataType.getDateStringValue((Integer) value), StandardCharsets.UTF_8); + } + return value instanceof Binary + ? (Binary) value + : new Binary(String.valueOf(value), StandardCharsets.UTF_8); + } + + public static final class TimeValueAccumulatorStrategy { + private final TimeValueColumnUpdater columnUpdater; + private final TimeValueStatisticsUpdater statisticsUpdater; + private final TimeValueFinalSetter finalSetter; + private final BiConsumer resultWriter; + + private TimeValueAccumulatorStrategy( + final TimeValueColumnUpdater columnUpdater, + final TimeValueStatisticsUpdater statisticsUpdater, + final TimeValueFinalSetter finalSetter, + final BiConsumer resultWriter) { + this.columnUpdater = columnUpdater; + this.statisticsUpdater = statisticsUpdater; + this.finalSetter = finalSetter; + this.resultWriter = resultWriter; + } + + private static TimeValueAccumulatorStrategy unsupported() { + return new TimeValueAccumulatorStrategy(null, null, null, null); + } + + public boolean isSupported() { + return columnUpdater != null; + } + + public void addInput( + final TimeValueAccumulator accumulator, + final Column[] columns, + final BitMap bitMap, + final boolean stopAfterFirstValue, + final boolean bitmapMarksSelected) { + final int count = columns[0].getPositionCount(); + for (int i = 0; i < count; i++) { + if (bitMap != null && bitmapMarksSelected != bitMap.isMarked(i)) { + continue; + } + if (!columns[1].isNull(i)) { + columnUpdater.update(accumulator, columns[1], i, columns[0].getLong(i)); + if (stopAfterFirstValue) { + return; + } + } + } + } + + public void addIntermediate( + final TimeValueAccumulator accumulator, + final Column valueColumn, + final Column timeColumn) { + columnUpdater.update(accumulator, valueColumn, 0, timeColumn.getLong(0)); + } + + public void addStatistics( + final TimeValueAccumulator accumulator, + final Statistics statistics, + final boolean first) { + statisticsUpdater.update(accumulator, statistics, first); + } + + public void setFinal(final TimeValueAccumulator accumulator, final Column column) { + finalSetter.set(accumulator, column); + } + + public void writeResult(final ColumnBuilder builder, final TsPrimitiveType result) { + resultWriter.accept(builder, result); + } + } + + public static final class MaxMinByAccumulatorStrategy { + private final Type type; + private final PrimitiveColumnValueSetter xValueSetter; + private final boolean xSupported; + private final MaxMinByYColumnUpdater yColumnUpdater; + private final MaxMinByYBytesUpdater yBytesUpdater; + + private MaxMinByAccumulatorStrategy( + final Type type, + final boolean xSupported, + final MaxMinByYColumnUpdater yColumnUpdater, + final MaxMinByYBytesUpdater yBytesUpdater) { + this.type = type; + this.xValueSetter = + org.apache.iotdb.calc.utils.TypeServices.PRIMITIVE_COLUMN_VALUE_SETTER_SERVICE.call( + type); + this.xSupported = xSupported; + this.yColumnUpdater = yColumnUpdater; + this.yBytesUpdater = yBytesUpdater; + } + + private static MaxMinByAccumulatorStrategy unsupported(final Type type) { + return new MaxMinByAccumulatorStrategy(type, false, null, null); + } + + public boolean isXSupported() { + return xSupported; + } + + public boolean isYSupported() { + return yColumnUpdater != null; + } + + public void addInput( + final MaxMinByAccumulator accumulator, final Column[] columns, final BitMap bitMap) { + final int count = columns[0].getPositionCount(); + for (int i = 0; i < count; i++) { + if (bitMap != null && !bitMap.isMarked(i)) { + continue; + } + if (!columns[2].isNull(i)) { + yColumnUpdater.update(accumulator, columns[0].getLong(i), columns[2], i, columns[1], i); + } + } + } + + public void setXResult(final TsPrimitiveType result, final Column column, final int index) { + // Type.setTo copies the primitive into the column; aggregation must read the input instead. + xValueSetter.set(result, column, index); + } + + public void writeXResult(final ColumnBuilder builder, final TsPrimitiveType result) { + type.write(builder, result); + } + + public void writeSerializedValue( + final byte[] bytes, final int offset, final ColumnBuilder builder) { + type.write(builder, bytes, offset); + } + + public void updateIntermediate( + final MaxMinByAccumulator accumulator, + final long time, + final byte[] bytes, + final int offset) { + yBytesUpdater.update(accumulator, time, bytes, offset); + } + } + + public static final class EventWindowManagerProvider { + private static final EventWindowManagerProvider UNSUPPORTED = + new EventWindowManagerProvider(null, null); + + private final EventWindowManagerFactory equalFactory; + private final EventWindowManagerFactory variationFactory; + + private EventWindowManagerProvider( + final EventWindowManagerFactory equalFactory, + final EventWindowManagerFactory variationFactory) { + this.equalFactory = equalFactory; + this.variationFactory = variationFactory; + } + + private static EventWindowManagerProvider unsupported() { + return UNSUPPORTED; + } + + public VariationWindowManager createEqual( + final VariationWindowParameter parameter, final boolean ascending) { + if (equalFactory == null) { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages + .EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_EQUAL_EVENT_AGGREGATION_ARG_77E085EB, + parameter.getDataType())); + } + return equalFactory.create(parameter, ascending); + } + + public VariationWindowManager createVariation( + final VariationWindowParameter parameter, final boolean ascending) { + if (variationFactory == null) { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages + .EXCEPTION_UNSUPPORTED_DATA_TYPE_IN_VARIATION_EVENT_AGGREGATION_ARG_3D0F0388, + parameter.getDataType())); + } + return variationFactory.create(parameter, ascending); + } + } + + private static ExtremeValueAccumulatorStrategy extremeValueAccumulatorStrategy( + final ExtremeValueColumnUpdater columnUpdater, + final ExtremeValueStatisticsUpdater statisticsUpdater, + final ExtremeValueFinalSetter finalSetter, + final BiConsumer resultWriter) { + return new ExtremeValueAccumulatorStrategy( + columnUpdater, statisticsUpdater, finalSetter, resultWriter); + } + + public static final class ExtremeValueAccumulatorStrategy { + private final ExtremeValueColumnUpdater columnUpdater; + private final ExtremeValueStatisticsUpdater statisticsUpdater; + private final ExtremeValueFinalSetter finalSetter; + private final BiConsumer resultWriter; + + private ExtremeValueAccumulatorStrategy( + final ExtremeValueColumnUpdater columnUpdater, + final ExtremeValueStatisticsUpdater statisticsUpdater, + final ExtremeValueFinalSetter finalSetter, + final BiConsumer resultWriter) { + this.columnUpdater = columnUpdater; + this.statisticsUpdater = statisticsUpdater; + this.finalSetter = finalSetter; + this.resultWriter = resultWriter; + } + + private static ExtremeValueAccumulatorStrategy unsupported() { + return new ExtremeValueAccumulatorStrategy(null, null, null, null); + } + + public boolean isSupported() { + return columnUpdater != null; + } + + public void addInput( + final ExtremeValueAccumulator accumulator, final Column[] columns, final BitMap bitMap) { + final int count = columns[0].getPositionCount(); + for (int i = 0; i < count; i++) { + if (bitMap != null && !bitMap.isMarked(i)) { + continue; + } + if (!columns[1].isNull(i)) { + columnUpdater.update(accumulator, columns[1], i); + } + } + } + + public void addIntermediate(final ExtremeValueAccumulator accumulator, final Column column) { + columnUpdater.update(accumulator, column, 0); + } + + public void addStatistics( + final ExtremeValueAccumulator accumulator, final Object statisticsValue) { + statisticsUpdater.update(accumulator, statisticsValue); + } + + public void setFinal(final ExtremeValueAccumulator accumulator, final Column column) { + finalSetter.set(accumulator, column); + } + + public void writeResult(final ColumnBuilder builder, final TsPrimitiveType result) { + resultWriter.accept(builder, result); + } + } + + public static final TypeService OUTPUT_COLUMN_SIZE_PER_LINE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, DATE, INT64, TIMESTAMP, FLOAT, DOUBLE -> + () -> type.estimateValueSize() + Byte.BYTES; + case TEXT, BLOB, STRING, OBJECT -> + () -> StatisticsManager.getInstance().getMaxBinarySizeInBytes(); + case ROW, UNKNOWN, VECTOR -> + () -> { + throw new UnsupportedOperationException( + DataNodeQueryMessages.UNKNOWN_DATA_TYPE_2 + type.getTypeEnum()); + }; + }; + + public static final TypeService> MODE_ACCUMULATOR_PROVIDER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> BooleanModeAccumulator::new; + case TEXT, BLOB, STRING -> BinaryModeAccumulator::new; + case INT32, DATE -> IntModeAccumulator::new; + case INT64, TIMESTAMP -> LongModeAccumulator::new; + case FLOAT -> FloatModeAccumulator::new; + case DOUBLE -> DoubleModeAccumulator::new; + case OBJECT, ROW, UNKNOWN, VECTOR -> + () -> { + throw new IllegalArgumentException( + DataNodeQueryMessages.UNKNOWN_DATA_TYPE + type.getTypeEnum()); + }; + }; + + static { + EXTREME_VALUE_ACCUMULATOR_STRATEGY_SERVICE.check(); + TIME_VALUE_ACCUMULATOR_STRATEGY_SERVICE.check(); + MAX_MIN_BY_ACCUMULATOR_STRATEGY_SERVICE.check(); + EVENT_WINDOW_MANAGER_PROVIDER_SERVICE.check(); + OUTPUT_COLUMN_SIZE_PER_LINE_SERVICE.check(); + MODE_ACCUMULATOR_PROVIDER_SERVICE.check(); + } + + private Aggregation() { + // Utility class + } + + public interface ExtremeValueAccumulator { + TsPrimitiveType getResult(); + + void updateIntResult(int value); + + void updateLongResult(long value); + + void updateFloatResult(float value); + + void updateDoubleResult(double value); + + void updateBinaryResult(Binary value); + } + + public interface TimeValueAccumulator { + TsPrimitiveType getTimeValueResult(); + + void updateIntResult(int value, long time); + + void updateLongResult(long value, long time); + + void updateFloatResult(float value, long time); + + void updateDoubleResult(double value, long time); + + void updateBinaryResult(Binary value, long time); + + void updateBooleanResult(boolean value, long time); + } + + public interface MaxMinByAccumulator { + Column readXFromBytesIntermediateInput(byte[] bytes, int offset); + + void updateIntResult(long time, int yValue, Column xColumn, int xIndex); + + void updateLongResult(long time, long yValue, Column xColumn, int xIndex); + + void updateFloatResult(long time, float yValue, Column xColumn, int xIndex); + + void updateDoubleResult(long time, double yValue, Column xColumn, int xIndex); + + void updateBinaryResult(long time, Binary yValue, Column xColumn, int xIndex); + } + + @FunctionalInterface + private interface ExtremeValueColumnUpdater { + void update(ExtremeValueAccumulator accumulator, Column column, int index); + } + + @FunctionalInterface + private interface ExtremeValueStatisticsUpdater { + void update(ExtremeValueAccumulator accumulator, Object value); + } + + @FunctionalInterface + private interface ExtremeValueFinalSetter { + void set(ExtremeValueAccumulator accumulator, Column column); + } + + @FunctionalInterface + private interface TimeValueColumnUpdater { + void update(TimeValueAccumulator accumulator, Column column, int index, long time); + } + + @FunctionalInterface + private interface TimeValueStatisticsUpdater { + void update(TimeValueAccumulator accumulator, Statistics statistics, boolean first); + } + + @FunctionalInterface + private interface TimeValueFinalSetter { + void set(TimeValueAccumulator accumulator, Column column); + } + + @FunctionalInterface + private interface MaxMinByYColumnUpdater { + void update( + MaxMinByAccumulator accumulator, + long time, + Column yColumn, + int yIndex, + Column xColumn, + int xIndex); + } + + @FunctionalInterface + private interface MaxMinByYBytesUpdater { + void update(MaxMinByAccumulator accumulator, long time, byte[] bytes, int offset); + } + + @FunctionalInterface + private interface EventWindowManagerFactory { + VariationWindowManager create(VariationWindowParameter parameter, boolean ascending); + } + } + + public static final class ValueConversion { + + public static final TypeService> VALUE_PARSER_NO_EXCEPTION_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> Boolean::parseBoolean; + case INT32 -> TypeServices::parseInteger; + case INT64 -> TypeServices::parseLong; + case FLOAT -> TypeServices::parseFloat; + case DOUBLE -> TypeServices::parseDouble; + case TEXT -> TypeServices::parseText; + case TIMESTAMP -> TypeServices::parseTimestamp; + case DATE -> TypeServices::parseDate; + case BLOB -> TypeServices::parseBlob; + case STRING -> TypeServices::parseString; + case OBJECT, ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(CalcMessages.UNKNOWN_DATATYPE + type) + .setChecked(true); + }; + + public static final TypeService VALUE_PARSER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (value, zoneId) -> CommonUtils.parseBoolean(value); + case INT32 -> + (value, zoneId) -> { + try { + return Integer.parseInt(StringUtils.trim(value)); + } catch (final NumberFormatException e) { + throw inconsistentValueException(value, type); + } + }; + case INT64 -> + (value, zoneId) -> { + try { + return Long.parseLong(StringUtils.trim(value)); + } catch (final NumberFormatException e) { + throw inconsistentValueException(value, type); + } + }; + case TIMESTAMP -> + (value, zoneId) -> { + try { + return TypeInferenceUtils.isNumber(value) + ? Long.parseLong(value) + : DataNodeDateTimeUtils.parseDateTimeExpressionToLong( + StringUtils.trim(value), zoneId); + } catch (final Throwable e) { + throw new NumberFormatException( + String.format( + DataNodeMiscMessages + .EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_ARG_REGISTERED_ARG_BECAUSE_ARG_C8637858, + value, + type.getTypeEnum(), + e.getMessage())); + } + }; + case DATE -> (value, zoneId) -> CommonUtils.parseIntFromString(value); + case FLOAT -> + (value, zoneId) -> { + final float result; + try { + result = Float.parseFloat(value); + } catch (final NumberFormatException e) { + throw inconsistentValueException(value, type); + } + if (Float.isInfinite(result)) { + throw new NumberFormatException(DataNodeMiscMessages.INPUT_FLOAT_INFINITY); + } + return result; + }; + case DOUBLE -> + (value, zoneId) -> { + final double result; + try { + result = Double.parseDouble(value); + } catch (final NumberFormatException e) { + throw inconsistentValueException(value, type); + } + if (Double.isInfinite(result)) { + throw new NumberFormatException(DataNodeMiscMessages.INPUT_DOUBLE_INFINITY); + } + return result; + }; + case TEXT, STRING -> + (value, zoneId) -> + new Binary(stripQuotesIfPresent(value), TSFileConfig.STRING_CHARSET); + case BLOB -> + (value, zoneId) -> + new Binary( + CommonUtils.parseBlobStringToByteArray(stripQuotesIfPresent(value))); + case OBJECT -> + (value, zoneId) -> { + throw inconsistentValueException(value, type); + }; + case ROW, UNKNOWN, VECTOR -> + (value, zoneId) -> { + throw new QueryProcessException( + DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()); + }; + }; + + public static final TypeService> AUTO_CAST_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + targetType -> + targetType == TSDataType.INT64 + || targetType == TSDataType.FLOAT + || targetType == TSDataType.DOUBLE; + case INT64, FLOAT -> targetType -> targetType == TSDataType.DOUBLE; + case BOOLEAN, DATE, TIMESTAMP, DOUBLE, TEXT, BLOB, STRING, OBJECT -> + targetType -> false; + case ROW, UNKNOWN, VECTOR -> + targetType -> { + throw new IllegalArgumentException( + DataNodeMiscMessages.UNKNOWN_DATA_TYPE + type.getTypeEnum()); + }; + }; + + public static final TypeService>> + PREPARED_PARAMETER_LITERAL_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + value -> { + final String text = Boolean.toString((Boolean) value); + return new Pair<>(new BooleanLiteral(text), text); + }; + case INT32, INT64 -> + value -> { + final String text = String.valueOf(value); + return new Pair<>(new LongLiteral(text), text); + }; + case FLOAT -> + value -> new Pair<>(new DoubleLiteral((Float) value), String.valueOf(value)); + case DOUBLE -> + value -> new Pair<>(new DoubleLiteral((Double) value), String.valueOf(value)); + case TEXT, STRING -> + value -> { + final String text = (String) value; + return new Pair<>( + new StringLiteral(text), "'" + text.replace("'", "''") + "'"); + }; + case BLOB -> + value -> { + final BinaryLiteral literal = new BinaryLiteral((byte[]) value); + return new Pair<>(literal, "X'" + literal.toHexString() + "'"); + }; + case DATE, TIMESTAMP, OBJECT, ROW, UNKNOWN, VECTOR -> + value -> { + throw new IllegalArgumentException( + DataNodeMiscMessages.UNKNOWN_PARAMETER_TYPE + type.getTypeEnum()); + }; + }; + + static { + VALUE_PARSER_NO_EXCEPTION_SERVICE.check(); + VALUE_PARSER_SERVICE.check(); + AUTO_CAST_SERVICE.check(); + PREPARED_PARAMETER_LITERAL_SERVICE.check(); + } + + private static NumberFormatException inconsistentValueException( + final String value, final Type type) { + return new NumberFormatException( + String.format( + DataNodeMiscMessages + .EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_ARG_REGISTERED_ARG_0EF32FD3, + value, + type.getTypeEnum())); + } + + private static String stripQuotesIfPresent(final String value) { + if ((value.startsWith(SqlConstant.QUOTE) && value.endsWith(SqlConstant.QUOTE)) + || (value.startsWith(SqlConstant.DQUOTE) && value.endsWith(SqlConstant.DQUOTE))) { + return value.length() == 1 ? value : value.substring(1, value.length() - 1); + } + return value; + } + + @FunctionalInterface + public interface ContextualValueParser { + Object parse(String value, ZoneId zoneId) throws QueryProcessException; + } + + private ValueConversion() { + // Utility class + } + } + + public static final class StorageEngine { + + public static final TypeService> + INSERT_ROW_SERIALIZED_VALUE_SIZE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> value -> Byte.BYTES; + case INT32, DATE, FLOAT -> value -> Integer.BYTES; + case INT64, TIMESTAMP, DOUBLE -> value -> Long.BYTES; + case TEXT, STRING, BLOB, OBJECT -> + value -> ReadWriteIOUtils.sizeToWrite((Binary) value); + case ROW, UNKNOWN, VECTOR -> + value -> { + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()); + }; + }; + + public static final TypeService + INSERT_TABLET_SERIALIZED_COLUMN_SIZE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (column, start, end) -> (end - start) * Byte.BYTES; + case INT32, DATE, FLOAT -> (column, start, end) -> (end - start) * Integer.BYTES; + case INT64, TIMESTAMP, DOUBLE -> + (column, start, end) -> (end - start) * Long.BYTES; + case TEXT, STRING, BLOB, OBJECT -> + (column, start, end) -> { + int size = 0; + Binary[] values = (Binary[]) column; + // Tablet and WAL use a length prefix without TsFile's presence byte. + // Count only the selected range, including placeholders for null values. + for (int i = start; i < end; i++) { + byte[] bytes = + values == null || values[i] == null ? null : values[i].getValues(); + size += Integer.BYTES + (bytes == null ? 0 : bytes.length); + } + return size; + }; + case ROW, UNKNOWN, VECTOR -> + (column, start, end) -> { + throw new UnSupportedDataTypeException( + String.format( + DataNodeQueryMessages + .QUERY_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, + type.getTypeEnum())); + }; + }; + + public static final TypeService + RAW_ARRAY_BYTE_BUFFER_SERIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, DATE, INT64, TIMESTAMP, FLOAT, DOUBLE -> + type::serializeArray; + case TEXT, BLOB, STRING, OBJECT -> + (column, rows, buffer) -> { + Binary[] values = (Binary[]) column; + // Preserve the Tablet wire format: nullness lives in the separate bitmap. + for (int i = 0; i < rows; i++) { + if (values[i] != null && values[i].getValues() != null) { + ReadWriteIOUtils.write(values[i], buffer); + } else { + buffer.putInt(0); + } + } + }; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(type.getTypeEnum().name()) + .setChecked(true); + }; + + public static final TypeService + RAW_ARRAY_OUTPUT_STREAM_SERIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, DATE, INT64, TIMESTAMP, FLOAT, DOUBLE -> + type::serializeArray; + case TEXT, BLOB, STRING, OBJECT -> + (column, rows, stream) -> { + Binary[] values = (Binary[]) column; + // Match the ByteBuffer writer, including zero-length null placeholders. + for (int i = 0; i < rows; i++) { + if (values[i] != null && values[i].getValues() != null) { + ReadWriteIOUtils.write(values[i], stream); + } else { + stream.writeInt(0); + } + } + }; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(type.getTypeEnum().name()) + .setChecked(true); + }; + + public static final TypeService TABLET_PLAIN_FAST_PATH_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, DATE, INT32, TIMESTAMP, INT64, FLOAT, DOUBLE, STRING, BLOB, TEXT -> + true; + case ROW, VECTOR, UNKNOWN, OBJECT -> false; + }; + + public static final TypeService + ALIGNED_TV_LIST_COLUMN_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (writer, time, list, row, column, isNull) -> + writer.writeByColumn( + time, !isNull && list.getBooleanByValueIndex(row, column), isNull); + case INT32, DATE -> + (writer, time, list, row, column, isNull) -> + writer.writeByColumn( + time, isNull ? 0 : list.getIntByValueIndex(row, column), isNull); + case INT64, TIMESTAMP -> + (writer, time, list, row, column, isNull) -> + writer.writeByColumn( + time, isNull ? 0L : list.getLongByValueIndex(row, column), isNull); + case FLOAT -> + (writer, time, list, row, column, isNull) -> + writer.writeByColumn( + time, isNull ? 0F : list.getFloatByValueIndex(row, column), isNull); + case DOUBLE -> + (writer, time, list, row, column, isNull) -> + writer.writeByColumn( + time, isNull ? 0D : list.getDoubleByValueIndex(row, column), isNull); + case TEXT, STRING, BLOB, OBJECT -> + (writer, time, list, row, column, isNull) -> + writer.writeByColumn( + time, + isNull ? null : list.getBinaryByValueIndex(row, column), + isNull); + // Preserve the flush path's no-op behavior for non-value types. + case ROW, UNKNOWN, VECTOR -> (writer, time, list, row, column, isNull) -> {}; + }; + + private static final Binary EMPTY_BINARY = new Binary("", StandardCharsets.UTF_8); + + public static final TypeService + CHUNK_METADATA_STATISTICS_CONVERTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, INT64, TIMESTAMP, FLOAT, DOUBLE, BOOLEAN -> + StorageEngine::convertNumericOrBooleanStatistics; + case DATE -> StorageEngine::convertDateStatistics; + case STRING -> StorageEngine::convertStringStatistics; + case TEXT -> StorageEngine::convertTextStatistics; + case BLOB -> StorageEngine::convertBlobStatistics; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (chunkMetadata, targetDataType, statistics) -> statistics; + }; + + public static final TypeService> + WAL_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (value, buffer) -> WALWriteUtils.write((Boolean) value, buffer); + case INT32, DATE -> + (value, buffer) -> WALWriteUtils.write((Integer) value, buffer); + case INT64, TIMESTAMP -> + (value, buffer) -> WALWriteUtils.write((Long) value, buffer); + case FLOAT -> (value, buffer) -> WALWriteUtils.write((Float) value, buffer); + case DOUBLE -> (value, buffer) -> WALWriteUtils.write((Double) value, buffer); + case TEXT, BLOB, STRING, OBJECT -> + (value, buffer) -> WALWriteUtils.write((Binary) value, buffer); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + // Unmaterialized blocks still write zero placeholders; the bitmap carries nullness in WAL. + public static final TypeService WAL_ARRAY_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (column, buffer, start, end) -> { + int[] values = (int[]) column; + for (int i = start; i < end; i++) { + buffer.putInt(values == null ? 0 : values[i]); + } + }; + case INT64, TIMESTAMP -> + (column, buffer, start, end) -> { + long[] values = (long[]) column; + for (int i = start; i < end; i++) { + buffer.putLong(values == null ? 0L : values[i]); + } + }; + case FLOAT -> + (column, buffer, start, end) -> { + float[] values = (float[]) column; + for (int i = start; i < end; i++) { + buffer.putFloat(values == null ? 0F : values[i]); + } + }; + case DOUBLE -> + (column, buffer, start, end) -> { + double[] values = (double[]) column; + for (int i = start; i < end; i++) { + buffer.putDouble(values == null ? 0D : values[i]); + } + }; + case BOOLEAN -> + (column, buffer, start, end) -> { + boolean[] values = (boolean[]) column; + for (int i = start; i < end; i++) { + buffer.put((byte) (values != null && values[i] ? 1 : 0)); + } + }; + case TEXT, BLOB, STRING, OBJECT -> + (column, buffer, start, end) -> { + Binary[] values = (Binary[]) column; + for (int i = start; i < end; i++) { + if (values != null && values[i] != null && values[i].getValues() != null) { + WALWriteUtils.write(values[i], buffer); + } else { + buffer.putInt(0); + } + } + }; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService DECODED_VALUE_CHUNK_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (writer, time, stream, isNull) -> + writer.write(time, isNull ? 0 : ReadWriteIOUtils.readInt(stream), isNull); + case INT64, TIMESTAMP -> + (writer, time, stream, isNull) -> + writer.write(time, isNull ? 0L : ReadWriteIOUtils.readLong(stream), isNull); + case FLOAT -> + (writer, time, stream, isNull) -> + writer.write(time, isNull ? 0F : ReadWriteIOUtils.readFloat(stream), isNull); + case DOUBLE -> + (writer, time, stream, isNull) -> + writer.write(time, isNull ? 0D : ReadWriteIOUtils.readDouble(stream), isNull); + case BOOLEAN -> + (writer, time, stream, isNull) -> + writer.write(time, !isNull && ReadWriteIOUtils.readBool(stream), isNull); + case TEXT, BLOB, STRING, OBJECT -> + (writer, time, stream, isNull) -> + writer.write( + time, isNull ? null : ReadWriteIOUtils.readBinary(stream), isNull); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService + SEGMENTED_ARRAY_SERIALIZED_SIZE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (valueArrays, rowCount, arraySize) -> rowCount * Byte.BYTES; + case INT32, DATE -> + (valueArrays, rowCount, arraySize) -> rowCount * Integer.BYTES; + case INT64, TIMESTAMP -> + (valueArrays, rowCount, arraySize) -> rowCount * Long.BYTES; + case FLOAT -> (valueArrays, rowCount, arraySize) -> rowCount * Float.BYTES; + case DOUBLE -> (valueArrays, rowCount, arraySize) -> rowCount * Double.BYTES; + case TEXT, BLOB, STRING, OBJECT -> + (valueArrays, rowCount, arraySize) -> { + int size = 0; + int remaining = rowCount; + for (Object valueArray : valueArrays) { + int length = Math.min(remaining, arraySize); + // WAL uses length-prefixed binaries, without TsFile array presence bytes. + Binary[] binaryValues = (Binary[]) valueArray; + for (int i = 0; i < length; i++) { + Binary value = binaryValues == null ? null : binaryValues[i]; + size += Integer.BYTES; + if (value != null && value.getValues() != null) { + size += value.getLength(); + } + } + remaining -= length; + if (remaining == 0) { + break; + } + } + return size; + }; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService ARRAY_VALUE_COLUMN_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (builder, values, index, floatPrecision, encoding) -> + builder.writeBoolean(((boolean[]) values)[index]); + case INT32 -> + (builder, values, index, floatPrecision, encoding) -> + builder.writeInt(((int[]) values)[index]); + case DATE -> + (builder, values, index, floatPrecision, encoding) -> { + int value = ((int[]) values)[index]; + if (builder instanceof BinaryColumnBuilder) { + ((BinaryColumnBuilder) builder).writeDate(value); + } else { + builder.writeInt(value); + } + }; + case INT64, TIMESTAMP -> + (builder, values, index, floatPrecision, encoding) -> + builder.writeLong(((long[]) values)[index]); + case FLOAT -> + (builder, values, index, floatPrecision, encoding) -> { + float value = ((float[]) values)[index]; + if (encoding != null + && !Float.isNaN(value) + && (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF)) { + value = MathUtils.roundWithGivenPrecision(value, floatPrecision); + } + builder.writeFloat(value); + }; + case DOUBLE -> + (builder, values, index, floatPrecision, encoding) -> { + double value = ((double[]) values)[index]; + if (encoding != null + && !Double.isNaN(value) + && (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF)) { + value = MathUtils.roundWithGivenPrecision(value, floatPrecision); + } + builder.writeDouble(value); + }; + case TEXT, BLOB, STRING, OBJECT -> + (builder, values, index, floatPrecision, encoding) -> + builder.writeBinary(((Binary[]) values)[index]); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService BATCH_DATA_COLUMN_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (batchData, time, column, index) -> + batchData.putBoolean(time, type.getBoolean(column, index)); + case INT32, DATE -> + (batchData, time, column, index) -> + batchData.putInt(time, type.getInt(column, index)); + case INT64, TIMESTAMP -> + (batchData, time, column, index) -> + batchData.putLong(time, type.getLong(column, index)); + case FLOAT -> + (batchData, time, column, index) -> + batchData.putFloat(time, type.getFloat(column, index)); + case DOUBLE -> + (batchData, time, column, index) -> + batchData.putDouble(time, type.getDouble(column, index)); + case TEXT, BLOB, STRING, OBJECT -> + (batchData, time, column, index) -> + batchData.putBinary(time, type.getBinary(column, index)); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService> OBJECT_VALUE_SERIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> (value, stream) -> ReadWriteIOUtils.write((int) value, stream); + case INT64, TIMESTAMP -> + (value, stream) -> ReadWriteIOUtils.write((long) value, stream); + case FLOAT -> (value, stream) -> ReadWriteIOUtils.write((float) value, stream); + case DOUBLE -> (value, stream) -> ReadWriteIOUtils.write((double) value, stream); + case BOOLEAN -> (value, stream) -> ReadWriteIOUtils.write((boolean) value, stream); + case TEXT, BLOB, STRING, OBJECT -> + (value, stream) -> ReadWriteIOUtils.write((Binary) value, stream); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService> + TS_PRIMITIVE_VALUE_SERIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (value, stream) -> ReadWriteIOUtils.write(value.getInt(), stream); + case INT64, TIMESTAMP -> + (value, stream) -> ReadWriteIOUtils.write(value.getLong(), stream); + case FLOAT -> (value, stream) -> ReadWriteIOUtils.write(value.getFloat(), stream); + case DOUBLE -> + (value, stream) -> ReadWriteIOUtils.write(value.getDouble(), stream); + case BOOLEAN -> + (value, stream) -> ReadWriteIOUtils.write(value.getBoolean(), stream); + case TEXT, BLOB, STRING, OBJECT -> + (value, stream) -> ReadWriteIOUtils.write(value.getBinary(), stream); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService> + EMPTY_TS_PRIMITIVE_TYPE_FACTORY_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, INT64, FLOAT, DOUBLE -> type::getTsPrimitiveType; + case DATE -> Type.fromTsDataType(TSDataType.INT32)::getTsPrimitiveType; + case TIMESTAMP -> Type.fromTsDataType(TSDataType.INT64)::getTsPrimitiveType; + case TEXT, BLOB, STRING, OBJECT -> + () -> + Type.fromTsDataType(TSDataType.TEXT) + .getTsPrimitiveType(new Binary("", TSFileConfig.STRING_CHARSET)); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService> DEFAULT_ENCODING_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> IoTDBConfig::getDefaultBooleanEncoding; + case INT32, DATE -> IoTDBConfig::getDefaultInt32Encoding; + case INT64, TIMESTAMP -> IoTDBConfig::getDefaultInt64Encoding; + case FLOAT -> IoTDBConfig::getDefaultFloatEncoding; + case DOUBLE -> IoTDBConfig::getDefaultDoubleEncoding; + case TEXT, BLOB, STRING, OBJECT -> IoTDBConfig::getDefaultTextEncoding; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService TABLET_COLUMN_DECODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (decoder, buffer, rowCount, encoding) -> { + int[] values = new int[rowCount]; + // PlainEncoder uses var int, which may cause compatibility problems. + for (int i = 0; i < rowCount; i++) { + values[i] = + encoding == TSEncoding.PLAIN + ? ReadWriteIOUtils.readInt(buffer) + : decoder.readInt(buffer); + } + return values; + }; + case INT64, TIMESTAMP -> + (decoder, buffer, rowCount, encoding) -> { + long[] values = new long[rowCount]; + for (int i = 0; i < rowCount; i++) { + values[i] = decoder.readLong(buffer); + } + return values; + }; + case FLOAT -> + (decoder, buffer, rowCount, encoding) -> { + float[] values = new float[rowCount]; + for (int i = 0; i < rowCount; i++) { + values[i] = decoder.readFloat(buffer); + } + return values; + }; + case DOUBLE -> + (decoder, buffer, rowCount, encoding) -> { + double[] values = new double[rowCount]; + for (int i = 0; i < rowCount; i++) { + values[i] = decoder.readDouble(buffer); + } + return values; + }; + case BOOLEAN -> + (decoder, buffer, rowCount, encoding) -> { + boolean[] values = new boolean[rowCount]; + for (int i = 0; i < rowCount; i++) { + values[i] = decoder.readBoolean(buffer); + } + return values; + }; + case TEXT, BLOB, STRING -> + (decoder, buffer, rowCount, encoding) -> { + Binary[] values = new Binary[rowCount]; + // PlainEncoder uses var int, which may cause compatibility problems. + for (int i = 0; i < rowCount; i++) { + values[i] = + encoding == TSEncoding.PLAIN + ? ReadWriteIOUtils.readBinary(buffer) + : decoder.readBinary(buffer); + } + return values; + }; + case OBJECT, ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService DECODED_ARRAY_VALUE_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (values, index, stream) -> + ((int[]) values)[index] = ReadWriteIOUtils.readInt(stream); + case INT64, TIMESTAMP -> + (values, index, stream) -> + ((long[]) values)[index] = ReadWriteIOUtils.readLong(stream); + case FLOAT -> + (values, index, stream) -> + ((float[]) values)[index] = ReadWriteIOUtils.readFloat(stream); + case DOUBLE -> + (values, index, stream) -> + ((double[]) values)[index] = ReadWriteIOUtils.readDouble(stream); + case BOOLEAN -> + (values, index, stream) -> + ((boolean[]) values)[index] = ReadWriteIOUtils.readBool(stream); + case TEXT, BLOB, STRING, OBJECT -> + (values, index, stream) -> + ((Binary[]) values)[index] = ReadWriteIOUtils.readBinary(stream); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService DECODED_CHUNK_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (writer, time, stream) -> writer.write(time, ReadWriteIOUtils.readInt(stream)); + case INT64, TIMESTAMP -> + (writer, time, stream) -> writer.write(time, ReadWriteIOUtils.readLong(stream)); + case FLOAT -> + (writer, time, stream) -> writer.write(time, ReadWriteIOUtils.readFloat(stream)); + case DOUBLE -> + (writer, time, stream) -> writer.write(time, ReadWriteIOUtils.readDouble(stream)); + case BOOLEAN -> + (writer, time, stream) -> writer.write(time, ReadWriteIOUtils.readBool(stream)); + case TEXT, BLOB, STRING, OBJECT -> + (writer, time, stream) -> writer.write(time, ReadWriteIOUtils.readBinary(stream)); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE_2 + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService TV_LIST_ARRAY_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (tvList, times, values, bitMap, start, end) -> + tvList.putBooleans(times, (boolean[]) values, bitMap, start, end); + case INT32, DATE -> + (tvList, times, values, bitMap, start, end) -> + tvList.putInts(times, (int[]) values, bitMap, start, end); + case INT64, TIMESTAMP -> + (tvList, times, values, bitMap, start, end) -> + tvList.putLongs(times, (long[]) values, bitMap, start, end); + case FLOAT -> + (tvList, times, values, bitMap, start, end) -> + tvList.putFloats(times, (float[]) values, bitMap, start, end); + case DOUBLE -> + (tvList, times, values, bitMap, start, end) -> + tvList.putDoubles(times, (double[]) values, bitMap, start, end); + case TEXT, BLOB, STRING, OBJECT -> + (tvList, times, values, bitMap, start, end) -> + tvList.putBinaries(times, (Binary[]) values, bitMap, start, end); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService TV_LIST_OBJECT_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (tvList, time, value) -> tvList.putBoolean(time, (boolean) value); + case INT32, DATE -> (tvList, time, value) -> tvList.putInt(time, (int) value); + case INT64, TIMESTAMP -> (tvList, time, value) -> tvList.putLong(time, (long) value); + case FLOAT -> (tvList, time, value) -> tvList.putFloat(time, (float) value); + case DOUBLE -> (tvList, time, value) -> tvList.putDouble(time, (double) value); + case TEXT, BLOB, STRING -> + (tvList, time, value) -> tvList.putBinary(time, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()) + .setChecked(true); + }; + + private static final TVListProvider UNSUPPORTED_TV_LIST_PROVIDER = + new TVListProvider(() -> null, stream -> null, stream -> null); + + public static final TypeService TV_LIST_PROVIDER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + new TVListProvider( + BooleanTVList::newList, + BooleanTVList::deserialize, + BooleanTVList::deserializeWithoutBitMap); + case INT32 -> + new TVListProvider( + () -> IntTVList.newList(TSDataType.INT32), + stream -> IntTVList.deserialize(stream, TSDataType.INT32), + stream -> IntTVList.deserializeWithoutBitMap(stream, TSDataType.INT32)); + case DATE -> + new TVListProvider( + () -> IntTVList.newList(TSDataType.DATE), + stream -> IntTVList.deserialize(stream, TSDataType.DATE), + stream -> IntTVList.deserializeWithoutBitMap(stream, TSDataType.DATE)); + case INT64, TIMESTAMP -> + new TVListProvider( + LongTVList::newList, + LongTVList::deserialize, + LongTVList::deserializeWithoutBitMap); + case FLOAT -> + new TVListProvider( + FloatTVList::newList, + FloatTVList::deserialize, + FloatTVList::deserializeWithoutBitMap); + case DOUBLE -> + new TVListProvider( + DoubleTVList::newList, + DoubleTVList::deserialize, + DoubleTVList::deserializeWithoutBitMap); + case TEXT, BLOB, STRING, OBJECT -> + new TVListProvider( + BinaryTVList::newList, + BinaryTVList::deserialize, + BinaryTVList::deserializeWithoutBitMap); + case ROW, UNKNOWN, VECTOR -> UNSUPPORTED_TV_LIST_PROVIDER; + }; + + public static final TypeService TV_LIST_CHUNK_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (writer, time, tvList, index) -> { + writer.write(time, tvList.getBoolean(index)); + return 8L + 1L; + }; + case INT32, DATE -> + (writer, time, tvList, index) -> { + writer.write(time, tvList.getInt(index)); + return 8L + 4L; + }; + case INT64, TIMESTAMP -> + (writer, time, tvList, index) -> { + writer.write(time, tvList.getLong(index)); + return 8L + 8L; + }; + case FLOAT -> + (writer, time, tvList, index) -> { + writer.write(time, tvList.getFloat(index)); + return 8L + 4L; + }; + case DOUBLE -> + (writer, time, tvList, index) -> { + writer.write(time, tvList.getDouble(index)); + return 8L + 8L; + }; + case TEXT, BLOB, STRING, OBJECT -> + (writer, time, tvList, index) -> { + Binary value = tvList.getBinary(index); + writer.write(time, value); + return 8L + MemUtils.getBinarySize(value); + }; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService TV_LIST_BATCH_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (tvList, index, time, filter, builder, floatPrecision, encoding, pagination) -> { + boolean value = tvList.getBoolean(index); + if (filter != null && !filter.satisfyBoolean(time, value)) { + return false; + } + if (consumeOffset(pagination)) { + return true; + } + consumeLimit(pagination); + builder.getTimeColumnBuilder().writeLong(time); + builder.getColumnBuilder(0).writeBoolean(value); + builder.declarePosition(); + return true; + }; + case INT32, DATE -> + (tvList, index, time, filter, builder, floatPrecision, encoding, pagination) -> { + int value = tvList.getInt(index); + if (filter != null && !filter.satisfyInteger(time, value)) { + return false; + } + if (consumeOffset(pagination)) { + return true; + } + consumeLimit(pagination); + builder.getTimeColumnBuilder().writeLong(time); + builder.getColumnBuilder(0).writeInt(value); + builder.declarePosition(); + return true; + }; + case INT64, TIMESTAMP -> + (tvList, index, time, filter, builder, floatPrecision, encoding, pagination) -> { + long value = tvList.getLong(index); + if (filter != null && !filter.satisfyLong(time, value)) { + return false; + } + if (consumeOffset(pagination)) { + return true; + } + consumeLimit(pagination); + builder.getTimeColumnBuilder().writeLong(time); + builder.getColumnBuilder(0).writeLong(value); + builder.declarePosition(); + return true; + }; + case FLOAT -> + (tvList, index, time, filter, builder, floatPrecision, encoding, pagination) -> { + float value = tvList.getFloat(index); + if (!Float.isNaN(value) + && (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF)) { + value = MathUtils.roundWithGivenPrecision(value, floatPrecision); + } + if (filter != null && !filter.satisfyFloat(time, value)) { + return false; + } + if (consumeOffset(pagination)) { + return true; + } + consumeLimit(pagination); + builder.getTimeColumnBuilder().writeLong(time); + builder.getColumnBuilder(0).writeFloat(value); + builder.declarePosition(); + return true; + }; + case DOUBLE -> + (tvList, index, time, filter, builder, floatPrecision, encoding, pagination) -> { + double value = tvList.getDouble(index); + if (!Double.isNaN(value) + && (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF)) { + value = MathUtils.roundWithGivenPrecision(value, floatPrecision); + } + if (filter != null && !filter.satisfyDouble(time, value)) { + return false; + } + if (consumeOffset(pagination)) { + return true; + } + consumeLimit(pagination); + builder.getTimeColumnBuilder().writeLong(time); + builder.getColumnBuilder(0).writeDouble(value); + builder.declarePosition(); + return true; + }; + case TEXT, BLOB, STRING, OBJECT -> + (tvList, index, time, filter, builder, floatPrecision, encoding, pagination) -> { + Binary value = tvList.getBinary(index); + if (filter != null && !filter.satisfyBinary(time, value)) { + return false; + } + if (consumeOffset(pagination)) { + return true; + } + consumeLimit(pagination); + builder.getTimeColumnBuilder().writeLong(time); + builder.getColumnBuilder(0).writeBinary(value); + builder.declarePosition(); + return true; + }; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()) + .setChecked(true); + }; + + private static boolean consumeOffset(PaginationController paginationController) { + if (paginationController != null && paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return true; + } + return false; + } + + private static void consumeLimit(PaginationController paginationController) { + if (paginationController != null) { + paginationController.consumeLimit(); + } + } + + public static final TypeService ALIGNED_TV_LIST_CHUNK_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (writer, time, tvList, rowIndex, columnIndex, isNull) -> + writer.write( + time, + isNull ? false : tvList.getBooleanByValueIndex(rowIndex, columnIndex), + isNull); + case INT32, DATE -> + (writer, time, tvList, rowIndex, columnIndex, isNull) -> + writer.write( + time, + isNull ? 0 : tvList.getIntByValueIndex(rowIndex, columnIndex), + isNull); + case INT64, TIMESTAMP -> + (writer, time, tvList, rowIndex, columnIndex, isNull) -> + writer.write( + time, + isNull ? 0L : tvList.getLongByValueIndex(rowIndex, columnIndex), + isNull); + case FLOAT -> + (writer, time, tvList, rowIndex, columnIndex, isNull) -> + writer.write( + time, + isNull ? 0F : tvList.getFloatByValueIndex(rowIndex, columnIndex), + isNull); + case DOUBLE -> + (writer, time, tvList, rowIndex, columnIndex, isNull) -> + writer.write( + time, + isNull ? 0D : tvList.getDoubleByValueIndex(rowIndex, columnIndex), + isNull); + case TEXT, BLOB, STRING, OBJECT -> + (writer, time, tvList, rowIndex, columnIndex, isNull) -> + writer.write( + time, + isNull ? null : tvList.getBinaryByValueIndex(rowIndex, columnIndex), + isNull); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService VALUE_CHUNK_ARRAY_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (writer, time, values, index, isNull) -> + writer.write( + time, !isNull && values != null && ((boolean[]) values)[index], isNull); + case INT32, DATE -> + (writer, time, values, index, isNull) -> + writer.write( + time, isNull || values == null ? 0 : ((int[]) values)[index], isNull); + case INT64, TIMESTAMP -> + (writer, time, values, index, isNull) -> + writer.write( + time, isNull || values == null ? 0L : ((long[]) values)[index], isNull); + case FLOAT -> + (writer, time, values, index, isNull) -> + writer.write( + time, isNull || values == null ? 0F : ((float[]) values)[index], isNull); + case DOUBLE -> + (writer, time, values, index, isNull) -> + writer.write( + time, isNull || values == null ? 0D : ((double[]) values)[index], isNull); + case TEXT, STRING, BLOB, OBJECT -> + (writer, time, values, index, isNull) -> + writer.write( + time, + isNull || values == null ? null : ((Binary[]) values)[index], + isNull); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService + VALUE_CHUNK_ARRAY_BATCH_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (writer, times, values, nulls, offset, count) -> + writer.write(times, (boolean[]) values, nulls, count, offset); + case INT32, DATE -> + (writer, times, values, nulls, offset, count) -> + writer.write(times, (int[]) values, nulls, count, offset); + case INT64, TIMESTAMP -> + (writer, times, values, nulls, offset, count) -> + writer.write(times, (long[]) values, nulls, count, offset); + case FLOAT -> + (writer, times, values, nulls, offset, count) -> + writer.write(times, (float[]) values, nulls, count, offset); + case DOUBLE -> + (writer, times, values, nulls, offset, count) -> + writer.write(times, (double[]) values, nulls, count, offset); + case TEXT, STRING, BLOB, OBJECT -> + (writer, times, values, nulls, offset, count) -> + writer.write(times, (Binary[]) values, nulls, count, offset); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()) + .setChecked(true); + }; + + // Preserve the bitmap path so nulls do not require a temporary boolean array or value encoding. + public static final TypeService + VALUE_CHUNK_ARRAY_BITMAP_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (writer, times, values, bitMap, offset, count) -> + writer + .getPageWriter() + .write(times, (boolean[]) values, bitMap, offset, count, offset); + case INT32, DATE -> + (writer, times, values, bitMap, offset, count) -> + writer + .getPageWriter() + .write(times, (int[]) values, bitMap, offset, count, offset); + case INT64, TIMESTAMP -> + (writer, times, values, bitMap, offset, count) -> + writer + .getPageWriter() + .write(times, (long[]) values, bitMap, offset, count, offset); + case FLOAT -> + (writer, times, values, bitMap, offset, count) -> + writer + .getPageWriter() + .write(times, (float[]) values, bitMap, offset, count, offset); + case DOUBLE -> + (writer, times, values, bitMap, offset, count) -> + writer + .getPageWriter() + .write(times, (double[]) values, bitMap, offset, count, offset); + case TEXT, STRING, BLOB, OBJECT -> + (writer, times, values, bitMap, offset, count) -> + writer + .getPageWriter() + .write(times, (Binary[]) values, bitMap, offset, count, offset); + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService> PRIMITIVE_ARRAY_ALLOCATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, INT64, TIMESTAMP, FLOAT, DOUBLE, TEXT, BLOB, STRING, OBJECT -> + type::createArray; + case DATE -> Type.fromTsDataType(TSDataType.INT32)::createArray; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(type.getTypeEnum().name()) + .setChecked(true); + }; + + public static final TypeService + SOURCE_COLUMN_TO_TABLET_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (sourceType, sourceColumn, sourceIndex, targetColumn, targetIndex) -> + ((boolean[]) targetColumn)[targetIndex] = + sourceType.getBoolean(sourceColumn, sourceIndex); + case INT32, DATE -> + (sourceType, sourceColumn, sourceIndex, targetColumn, targetIndex) -> + ((int[]) targetColumn)[targetIndex] = + sourceType.getInt(sourceColumn, sourceIndex); + case INT64, TIMESTAMP -> + (sourceType, sourceColumn, sourceIndex, targetColumn, targetIndex) -> + ((long[]) targetColumn)[targetIndex] = + sourceType.getLong(sourceColumn, sourceIndex); + case FLOAT -> + (sourceType, sourceColumn, sourceIndex, targetColumn, targetIndex) -> + ((float[]) targetColumn)[targetIndex] = + sourceType.getFloat(sourceColumn, sourceIndex); + case DOUBLE -> + (sourceType, sourceColumn, sourceIndex, targetColumn, targetIndex) -> + ((double[]) targetColumn)[targetIndex] = + sourceType.getDouble(sourceColumn, sourceIndex); + case TEXT, BLOB, STRING -> + (sourceType, sourceColumn, sourceIndex, targetColumn, targetIndex) -> + ((Binary[]) targetColumn)[targetIndex] = + sourceType.getBinary(sourceColumn, sourceIndex); + case OBJECT, ROW, UNKNOWN, VECTOR -> + (sourceType, sourceColumn, sourceIndex, targetColumn, targetIndex) -> { + throw new UnSupportedDataTypeException( + DataNodeMiscMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()); + }; + }; + + public static final TypeService> TABLET_COLUMN_ALLOCATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, + INT32, + DATE, + INT64, + TIMESTAMP, + FLOAT, + DOUBLE, + TEXT, + BLOB, + STRING, + OBJECT -> + type::createArray; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(type.getTypeEnum().name()) + .setChecked(true); + }; + + public static final TypeService> EMPTY_TABLET_COLUMN_FACTORY_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, INT64, TIMESTAMP, FLOAT, DOUBLE -> type::createArray; + case DATE -> + size -> { + LocalDate[] values = (LocalDate[]) type.createArray(size); + Arrays.fill(values, LocalDate.of(1000, 1, 1)); + return values; + }; + case TEXT, BLOB, STRING -> + size -> { + Binary[] values = (Binary[]) type.createArray(size); + Arrays.fill(values, Binary.EMPTY_VALUE); + return values; + }; + case OBJECT, ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(type.getTypeEnum().name()) + .setChecked(true); + }; + + public static final TypeService WINDOW_VALUE_ARRAY_BUILDER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (window, list, begin, size) -> { + int[] values = new int[size]; + for (int i = 0; i < size; i++) { + values[i] = list.getIntByIndex(begin + i); + } + window.setIntValues(values); + }; + case INT64, TIMESTAMP -> + (window, list, begin, size) -> { + long[] values = new long[size]; + for (int i = 0; i < size; i++) { + values[i] = list.getLongByIndex(begin + i); + } + window.setLongValues(values); + }; + case FLOAT -> + (window, list, begin, size) -> { + float[] values = new float[size]; + for (int i = 0; i < size; i++) { + values[i] = list.getFloatByIndex(begin + i); + } + window.setFloatValues(values); + }; + case DOUBLE -> + (window, list, begin, size) -> { + double[] values = new double[size]; + for (int i = 0; i < size; i++) { + values[i] = list.getDoubleByIndex(begin + i); + } + window.setDoubleValues(values); + }; + case BOOLEAN -> + (window, list, begin, size) -> { + boolean[] values = new boolean[size]; + for (int i = 0; i < size; i++) { + values[i] = list.getBooleanByIndex(begin + i); + } + window.setBooleanValues(values); + }; + case TEXT, BLOB, STRING, OBJECT -> + (window, list, begin, size) -> { + Binary[] values = new Binary[size]; + for (int i = 0; i < size; i++) { + values[i] = list.getBinaryByIndex(begin + i); + } + window.setBinaryValues(values); + }; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(type.getTypeEnum().name()) + .setChecked(true); + }; + + public static final TypeService> + RAW_ARRAY_BYTE_BUFFER_DESERIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, INT64, TIMESTAMP, FLOAT, DOUBLE -> type::deserializeArray; + case TEXT, BLOB, STRING, OBJECT -> + (buffer, size) -> { + // Tablet RPC has no per-value presence byte; nulls use a separate bitmap. + Binary[] values = new Binary[size]; + for (int i = 0; i < size; i++) { + values[i] = ReadWriteIOUtils.readBinary(buffer); + } + return values; + }; + case DATE -> Type.fromTsDataType(TSDataType.INT32)::deserializeArray; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(type.getTypeEnum().name()) + .setChecked(true); + }; + + public static final TypeService + RAW_ARRAY_INPUT_STREAM_DESERIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, INT64, TIMESTAMP, FLOAT, DOUBLE -> type::deserializeArray; + case TEXT, BLOB, STRING, OBJECT -> + (stream, size) -> { + // Match the ByteBuffer tablet reader, not TsFile's array wire format. + Binary[] values = new Binary[size]; + for (int i = 0; i < size; i++) { + values[i] = ReadWriteIOUtils.readBinary(stream); + } + return values; + }; + case DATE -> Type.fromTsDataType(TSDataType.INT32)::deserializeArray; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(type.getTypeEnum().name()) + .setChecked(true); + }; + + public static final TypeService ARRAY_VALUE_GETTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (array, index) -> ((boolean[]) array)[index]; + case INT32, DATE -> (array, index) -> ((int[]) array)[index]; + case INT64, TIMESTAMP -> (array, index) -> ((long[]) array)[index]; + case FLOAT -> (array, index) -> ((float[]) array)[index]; + case DOUBLE -> (array, index) -> ((double[]) array)[index]; + case TEXT, BLOB, STRING, OBJECT -> (array, index) -> ((Binary[]) array)[index]; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException(type.getTypeEnum().name()) + .setChecked(true); + }; + + static { + CHUNK_METADATA_STATISTICS_CONVERTER_SERVICE.check(); + INSERT_ROW_SERIALIZED_VALUE_SIZE_SERVICE.check(); + INSERT_TABLET_SERIALIZED_COLUMN_SIZE_SERVICE.check(); + RAW_ARRAY_BYTE_BUFFER_SERIALIZER_SERVICE.check(); + RAW_ARRAY_OUTPUT_STREAM_SERIALIZER_SERVICE.check(); + TABLET_PLAIN_FAST_PATH_SERVICE.check(); + ALIGNED_TV_LIST_COLUMN_WRITER_SERVICE.check(); + TV_LIST_ARRAY_WRITER_SERVICE.check(); + TV_LIST_OBJECT_WRITER_SERVICE.check(); + TV_LIST_PROVIDER_SERVICE.check(); + TV_LIST_CHUNK_WRITER_SERVICE.check(); + TV_LIST_BATCH_WRITER_SERVICE.check(); + ALIGNED_TV_LIST_CHUNK_WRITER_SERVICE.check(); + VALUE_CHUNK_ARRAY_WRITER_SERVICE.check(); + VALUE_CHUNK_ARRAY_BATCH_WRITER_SERVICE.check(); + VALUE_CHUNK_ARRAY_BITMAP_WRITER_SERVICE.check(); + PRIMITIVE_ARRAY_ALLOCATOR_SERVICE.check(); + SOURCE_COLUMN_TO_TABLET_VALUE_WRITER_SERVICE.check(); + TABLET_COLUMN_ALLOCATOR_SERVICE.check(); + EMPTY_TABLET_COLUMN_FACTORY_SERVICE.check(); + WINDOW_VALUE_ARRAY_BUILDER_SERVICE.check(); + RAW_ARRAY_BYTE_BUFFER_DESERIALIZER_SERVICE.check(); + RAW_ARRAY_INPUT_STREAM_DESERIALIZER_SERVICE.check(); + ARRAY_VALUE_GETTER_SERVICE.check(); + DECODED_VALUE_CHUNK_WRITER_SERVICE.check(); + SEGMENTED_ARRAY_SERIALIZED_SIZE_SERVICE.check(); + ARRAY_VALUE_COLUMN_WRITER_SERVICE.check(); + BATCH_DATA_COLUMN_WRITER_SERVICE.check(); + OBJECT_VALUE_SERIALIZER_SERVICE.check(); + TS_PRIMITIVE_VALUE_SERIALIZER_SERVICE.check(); + EMPTY_TS_PRIMITIVE_TYPE_FACTORY_SERVICE.check(); + DEFAULT_ENCODING_SERVICE.check(); + TABLET_COLUMN_DECODER_SERVICE.check(); + DECODED_ARRAY_VALUE_READER_SERVICE.check(); + DECODED_CHUNK_WRITER_SERVICE.check(); + } + + private static Statistics convertNumericOrBooleanStatistics( + IChunkMetadata chunkMetadata, TSDataType targetDataType, Statistics statistics) { + if (targetDataType == TSDataType.STRING) { + Binary[] binaryValues = new Binary[4]; + binaryValues[0] = + new Binary( + chunkMetadata.getStatistics().getFirstValue().toString(), StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + chunkMetadata.getStatistics().getLastValue().toString(), StandardCharsets.UTF_8); + if (chunkMetadata.getDataType() == TSDataType.BOOLEAN) { + binaryValues[2] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); + binaryValues[3] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); + } else { + binaryValues[2] = + new Binary( + chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8); + binaryValues[3] = + new Binary( + chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); + } + updateStatistics(statistics, chunkMetadata, binaryValues); + return statistics; + } + if (targetDataType == TSDataType.TEXT) { + Binary[] binaryValues = new Binary[2]; + if (chunkMetadata.getDataType() == TSDataType.BOOLEAN) { + binaryValues[0] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); + binaryValues[1] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); + } else { + binaryValues[0] = + new Binary( + chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); + } + updateStatistics(statistics, chunkMetadata, binaryValues); + return statistics; + } + return chunkMetadata.getStatistics(); + } + + private static Statistics convertDateStatistics( + IChunkMetadata chunkMetadata, TSDataType targetDataType, Statistics statistics) { + if (targetDataType != TSDataType.STRING && targetDataType != TSDataType.TEXT) { + return statistics; + } + int valueCount = targetDataType == TSDataType.STRING ? 4 : 2; + Binary[] binaryValues = new Binary[valueCount]; + binaryValues[0] = toDateBinary(chunkMetadata.getStatistics().getFirstValue()); + binaryValues[1] = toDateBinary(chunkMetadata.getStatistics().getLastValue()); + if (targetDataType == TSDataType.STRING) { + binaryValues[2] = toDateBinary(chunkMetadata.getStatistics().getMinValue()); + binaryValues[3] = toDateBinary(chunkMetadata.getStatistics().getMaxValue()); + } + updateStatistics(statistics, chunkMetadata, binaryValues); + return statistics; + } + + private static Statistics convertStringStatistics( + IChunkMetadata chunkMetadata, TSDataType targetDataType, Statistics statistics) { + if (targetDataType == TSDataType.TEXT) { + Binary[] binaryValues = { + new Binary( + chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8), + new Binary(chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8) + }; + updateStatistics(statistics, chunkMetadata, binaryValues); + return statistics; + } + if (targetDataType == TSDataType.BLOB) { + statistics.update( + chunkMetadata.getStatistics().getStartTime(), + new Binary( + chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8)); + statistics.update( + chunkMetadata.getStatistics().getEndTime(), + new Binary( + chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8)); + return statistics; + } + return chunkMetadata.getStatistics(); + } + + private static Statistics convertTextStatistics( + IChunkMetadata chunkMetadata, TSDataType targetDataType, Statistics statistics) { + if (targetDataType == TSDataType.STRING) { + Binary[] binaryValues = { + (Binary) chunkMetadata.getStatistics().getFirstValue(), + (Binary) chunkMetadata.getStatistics().getLastValue() + }; + updateStatistics(statistics, chunkMetadata, binaryValues); + return statistics; + } + if (targetDataType == TSDataType.BLOB) { + statistics.update(chunkMetadata.getStatistics().getStartTime(), EMPTY_BINARY); + statistics.update(chunkMetadata.getStatistics().getEndTime(), EMPTY_BINARY); + return statistics; + } + return chunkMetadata.getStatistics(); + } + + private static Statistics convertBlobStatistics( + IChunkMetadata chunkMetadata, TSDataType targetDataType, Statistics statistics) { + if (targetDataType == TSDataType.STRING || targetDataType == TSDataType.TEXT) { + updateStatistics(statistics, chunkMetadata, new Binary[] {EMPTY_BINARY, EMPTY_BINARY}); + return statistics; + } + return chunkMetadata.getStatistics(); + } + + private static Binary toDateBinary(Object value) { + return new Binary(TSDataType.getDateStringValue((Integer) value), StandardCharsets.UTF_8); + } + + private static void updateStatistics( + Statistics statistics, IChunkMetadata chunkMetadata, Binary[] binaryValues) { + long[] longValues = new long[binaryValues.length]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + Arrays.fill(longValues, 2, longValues.length, longValues[1]); + statistics.update(longValues, binaryValues, binaryValues.length); + } + + private StorageEngine() { + // Utility class + } + } + + public static final class Predicate { + + public static final TypeService>> + CONVERT_PREDICATE_VALUE_PARSER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> Integer::valueOf; + case INT64, TIMESTAMP -> Long::valueOf; + case FLOAT -> Float::valueOf; + case DOUBLE -> Double::valueOf; + case BOOLEAN -> + valueString -> { + if (valueString.equalsIgnoreCase("true")) { + return Boolean.TRUE; + } else if (valueString.equalsIgnoreCase("false")) { + return Boolean.FALSE; + } + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages.EXCEPTION_ARG_CANNOT_BE_CAST_TO_ARG_28F0C5FC, + valueString, + type.getTypeEnum())); + }; + case BLOB -> valueString -> new Binary(BaseEncoding.base16().decode(valueString)); + case TEXT, STRING -> + valueString -> new Binary(valueString, TSFileConfig.STRING_CHARSET); + case DATE -> DateTimeUtils::parseDateExpressionToInt; + case OBJECT, ROW, UNKNOWN, VECTOR -> + throw new UnsupportedOperationException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, + type.getTypeEnum())); + }; + + public static final TypeService>> + RELATIONAL_CONVERT_PREDICATE_VALUE_PARSER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> value -> Integer.valueOf((int) getLongValue(value)); + case DATE -> value -> Integer.valueOf(((GenericLiteral) value).getValue()); + case INT64 -> value -> Long.valueOf(getLongValue(value)); + case TIMESTAMP -> TypeServices::getTimestampValue; + case FLOAT -> value -> Float.valueOf((float) getDoubleValue(value)); + case DOUBLE -> value -> Double.valueOf(getDoubleValue(value)); + case BOOLEAN -> value -> Boolean.valueOf(((BooleanLiteral) value).getValue()); + case TEXT, STRING -> + value -> + new Binary( + ((StringLiteral) value).getValue(), TSFileConfig.STRING_CHARSET); + case BLOB -> value -> new Binary(((BinaryLiteral) value).getValue()); + case OBJECT, ROW, UNKNOWN, VECTOR -> + throw new UnsupportedOperationException( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, + type.getTypeEnum())); + }; + + public static final TypeService> + UNCORRELATED_SCALAR_SUBQUERY_RESULT_LITERAL_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + column -> new LongLiteral(Long.toString(type.getInt(column, 0))); + case INT64, TIMESTAMP -> + column -> new LongLiteral(Long.toString(type.getLong(column, 0))); + case FLOAT -> column -> new FloatLiteral(type.getFloat(column, 0)); + case DOUBLE -> + column -> new DoubleLiteral(Double.toString(type.getDouble(column, 0))); + case BOOLEAN -> + column -> new BooleanLiteral(Boolean.toString(type.getBoolean(column, 0))); + case BLOB -> column -> new BinaryLiteral(type.getBinary(column, 0).toString()); + case TEXT, STRING -> + column -> new StringLiteral(type.getBinary(column, 0).toString()); + case OBJECT, ROW, UNKNOWN, VECTOR -> + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages + .EXCEPTION_UNSUPPORTED_DATA_TYPE_FOR_SCALAR_SUBQUERY_RESULT_ARG_D58CBB00, + type.getTypeEnum())); + }; + + private Predicate() { + // Utility class + } + } + + public static final class Pipe { + + public static final TypeService> OPC_UA_LAST_VALUE_CONVERTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case DATE -> + value -> + new DateTime( + new java.util.Date( + DateUtils.parseIntToDate(((Number) value).intValue()).getTime())); + case TIMESTAMP -> + value -> + new DateTime( + TimestampPrecisionUtils.currPrecision.toNanos( + ((Number) value).longValue()) + / 100L + + 116444736000000000L); + case TEXT, BLOB, STRING -> String::valueOf; + case BOOLEAN, INT32, INT64, FLOAT, DOUBLE -> Function.identity(); + case ROW, VECTOR, OBJECT, UNKNOWN -> + value -> { + throw new UnSupportedDataTypeException( + DataNodePipeMessages.UNSUPPORTED_DATATYPE + type.getTypeEnum()); + }; + }; + + public static final TypeService + SAME_TYPE_NUMERIC_OPERATOR_STRATEGY_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + sameTypeNumericOperatorStrategy( + TSDataType.INT32, + AbstractSameTypeNumericOperator::getIntValue, + (operator, stream) -> + ReadWriteIOUtils.write(operator.getIntValue(), stream), + (operator, buffer) -> + operator.setIntValue(ReadWriteIOUtils.readInt(buffer))); + case INT64 -> + sameTypeNumericOperatorStrategy( + TSDataType.INT64, + AbstractSameTypeNumericOperator::getLongValue, + (operator, stream) -> + ReadWriteIOUtils.write(operator.getLongValue(), stream), + (operator, buffer) -> + operator.setLongValue(ReadWriteIOUtils.readLong(buffer))); + case FLOAT -> + sameTypeNumericOperatorStrategy( + TSDataType.FLOAT, + AbstractSameTypeNumericOperator::getFloatValue, + (operator, stream) -> + ReadWriteIOUtils.write(operator.getFloatValue(), stream), + (operator, buffer) -> + operator.setFloatValue(ReadWriteIOUtils.readFloat(buffer))); + case DOUBLE -> + sameTypeNumericOperatorStrategy( + TSDataType.DOUBLE, + AbstractSameTypeNumericOperator::getDoubleValue, + (operator, stream) -> + ReadWriteIOUtils.write(operator.getDoubleValue(), stream), + (operator, buffer) -> + operator.setDoubleValue(ReadWriteIOUtils.readDouble(buffer))); + case BOOLEAN, DATE, TIMESTAMP, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + unsupportedSameTypeNumericOperatorStrategy( + TSDataType.valueOf(type.getTypeEnum().name())); + }; + + private static SameTypeNumericOperatorStrategy sameTypeNumericOperatorStrategy( + final TSDataType dataType, + final Function valueGetter, + final SameTypeNumericOperatorSerializer serializer, + final SameTypeNumericOperatorDeserializer deserializer) { + return new SameTypeNumericOperatorStrategy(dataType, valueGetter, serializer, deserializer); + } + + private static SameTypeNumericOperatorStrategy unsupportedSameTypeNumericOperatorStrategy( + final TSDataType dataType) { + return new SameTypeNumericOperatorStrategy( + dataType, + operator -> null, + (operator, stream) -> { + throw new IOException( + String.format(DataNodePipeMessages.UNSUPPORTED_OUTPUT_DATATYPE_FMT, dataType)); + }, + (operator, buffer) -> { + throw new IOException( + String.format(DataNodePipeMessages.UNSUPPORTED_OUTPUT_DATATYPE_FMT, dataType)); + }); + } + + public static final class SameTypeNumericOperatorStrategy { + private final TSDataType dataType; + private final Function valueGetter; + private final SameTypeNumericOperatorSerializer serializer; + private final SameTypeNumericOperatorDeserializer deserializer; + + private SameTypeNumericOperatorStrategy( + final TSDataType dataType, + final Function valueGetter, + final SameTypeNumericOperatorSerializer serializer, + final SameTypeNumericOperatorDeserializer deserializer) { + this.dataType = dataType; + this.valueGetter = valueGetter; + this.serializer = serializer; + this.deserializer = deserializer; + } + + public Pair getResult(final AbstractSameTypeNumericOperator operator) { + final Object value = valueGetter.apply(operator); + return value == null ? null : new Pair<>(dataType, value); + } + + public void serialize( + final AbstractSameTypeNumericOperator operator, final DataOutputStream stream) + throws IOException { + serializer.serialize(operator, stream); + } + + public void deserialize( + final AbstractSameTypeNumericOperator operator, final ByteBuffer buffer) + throws IOException { + deserializer.deserialize(operator, buffer); + } + } + + @FunctionalInterface + private interface SameTypeNumericOperatorSerializer { + void serialize(AbstractSameTypeNumericOperator operator, DataOutputStream stream) + throws IOException; + } + + @FunctionalInterface + private interface SameTypeNumericOperatorDeserializer { + void deserialize(AbstractSameTypeNumericOperator operator, ByteBuffer buffer) + throws IOException; + } + + public static final TypeService OPC_DA_VARIANT_TYPE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> Variant.VT_BOOL; + case INT32 -> Variant.VT_I4; + case INT64 -> Variant.VT_I8; + case DATE, TIMESTAMP -> Variant.VT_DATE; + case FLOAT -> Variant.VT_R4; + case DOUBLE -> Variant.VT_R8; + // Note that "Variant" does not support "VT_BLOB" data, and not all the DA servers + // support this, thus we use "VT_BSTR" to substitute. + case TEXT, STRING, BLOB, OBJECT -> Variant.VT_BSTR; + case ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + DataNodePipeMessages.UNSUPPORTED_DATATYPE + type.getTypeEnum()) + .setChecked(true); + }; + + public static final TypeService OPC_DA_TABLET_VALUE_SETTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (value, column, rowIndex) -> { + value.setValue( + Variant.VT_BOOL, new OaIdl.VARIANT_BOOL(((boolean[]) column)[rowIndex])); + return null; + }; + case INT32 -> + (value, column, rowIndex) -> { + value.setValue(Variant.VT_I4, new WinDef.LONG(((int[]) column)[rowIndex])); + return null; + }; + case DATE -> + (value, column, rowIndex) -> { + value.setValue( + Variant.VT_DATE, + new OaIdl.DATE(Date.valueOf(((LocalDate[]) column)[rowIndex]))); + return null; + }; + case INT64 -> + (value, column, rowIndex) -> { + value.setValue(Variant.VT_I8, new WinDef.LONGLONG(((long[]) column)[rowIndex])); + return null; + }; + case TIMESTAMP -> + (value, column, rowIndex) -> { + value.setValue( + Variant.VT_DATE, + new OaIdl.DATE(new java.util.Date(((long[]) column)[rowIndex]))); + return null; + }; + case FLOAT -> + (value, column, rowIndex) -> { + value.setValue(Variant.VT_R4, ((float[]) column)[rowIndex]); + return null; + }; + case DOUBLE -> + (value, column, rowIndex) -> { + value.setValue(Variant.VT_R8, ((double[]) column)[rowIndex]); + return null; + }; + case TEXT, STRING, BLOB, OBJECT -> + (value, column, rowIndex) -> { + final WTypes.BSTR bstr = + OleAuto.INSTANCE.SysAllocString(((Binary[]) column)[rowIndex].toString()); + value.setValue(Variant.VT_BSTR, bstr); + return bstr; + }; + case ROW, UNKNOWN, VECTOR -> + (value, column, rowIndex) -> { + throw new UnSupportedDataTypeException( + DataNodePipeMessages.UNSUPPORTED_DATATYPE + type.getTypeEnum()); + }; + }; + + public static final TypeService> OPC_UA_VALUE_STRINGIFIER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, BLOB, STRING -> Object::toString; + case DATE -> + value -> ((LocalDate) value).atStartOfDay(ZoneId.systemDefault()).toString(); + case TIMESTAMP -> value -> DateTimeUtils.convertLongToDate((long) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> + value -> { + throw new PipeRuntimeNonCriticalException( + DataNodePipeMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()); + }; + }; + + public static final TypeService> + CUSTOMIZED_INTERMEDIATE_RESULT_TO_INT_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> value -> (int) value; + case DATE -> value -> DateUtils.parseDateExpressionToInt((LocalDate) value); + case INT64, TIMESTAMP -> value -> (int) (long) value; + case FLOAT -> value -> (int) (float) value; + case DOUBLE -> value -> (int) (double) value; + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + value -> { + throw new UnsupportedOperationException( + String.format( + "The type %s cannot be casted to int.", type.getTypeEnum())); + }; + }; + + public static final TypeService> + CUSTOMIZED_INTERMEDIATE_RESULT_TO_LONG_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> value -> (int) value; + case DATE -> value -> DateUtils.parseDateExpressionToInt((LocalDate) value); + case INT64, TIMESTAMP -> value -> (long) value; + case FLOAT -> value -> (long) (float) value; + case DOUBLE -> value -> (long) (double) value; + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + value -> { + throw new UnsupportedOperationException( + String.format( + "The type %s cannot be casted to long.", type.getTypeEnum())); + }; + }; + + public static final TypeService> + CUSTOMIZED_INTERMEDIATE_RESULT_TO_FLOAT_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> value -> (float) (int) value; + case DATE -> + value -> (float) DateUtils.parseDateExpressionToInt((LocalDate) value); + case INT64, TIMESTAMP -> value -> (float) (long) value; + case FLOAT -> value -> (float) value; + case DOUBLE -> value -> (float) (double) value; + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + value -> { + throw new UnsupportedOperationException( + String.format( + "The type %s cannot be casted to float.", type.getTypeEnum())); + }; + }; + + public static final TypeService> + CUSTOMIZED_INTERMEDIATE_RESULT_TO_DOUBLE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> value -> (int) value; + case DATE -> value -> DateUtils.parseDateExpressionToInt((LocalDate) value); + case INT64, TIMESTAMP -> value -> (long) value; + case FLOAT -> value -> (float) value; + case DOUBLE -> value -> (double) value; + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + value -> { + throw new UnsupportedOperationException( + String.format( + "The type %s cannot be casted to double.", type.getTypeEnum())); + }; + }; + + public static final TypeService> + CUSTOMIZED_INTERMEDIATE_RESULT_TO_STRING_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> value -> Boolean.toString((boolean) value); + case INT32 -> value -> Integer.toString((int) value); + case DATE -> value -> ((LocalDate) value).toString(); + case INT64 -> value -> Long.toString((long) value); + case TIMESTAMP -> + value -> + RpcUtils.formatDatetime( + RpcUtils.DEFAULT_TIME_FORMAT, + CommonDescriptor.getInstance().getConfig().getTimestampPrecision(), + (long) value, + ZoneId.systemDefault()); + case FLOAT -> value -> Float.toString((float) value); + case DOUBLE -> value -> Double.toString((double) value); + case TEXT, STRING -> value -> (String) value; + case BLOB -> + value -> + BytesUtils.parseBlobByteArrayToString( + ((org.apache.iotdb.pipe.api.type.Binary) value).getValues()); + case OBJECT, ROW, UNKNOWN, VECTOR -> + value -> { + throw new UnsupportedOperationException( + String.format( + "The type %s cannot be casted to string.", type.getTypeEnum())); + }; + }; + + public static final TypeService> AGGREGATE_TABLET_COLUMN_ALLOCATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, + INT32, + DATE, + INT64, + TIMESTAMP, + FLOAT, + DOUBLE, + TEXT, + BLOB, + STRING, + OBJECT -> + type::createArray; + case ROW, UNKNOWN, VECTOR -> + size -> { + throw new UnsupportedOperationException( + String.format( + DataNodePipeMessages.UNSUPPORTED_OUTPUT_DATATYPE_FMT, + type.getTypeEnum())); + }; + }; + + public static final TypeService + AGGREGATE_TABLET_COLUMN_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, + INT32, + DATE, + INT64, + TIMESTAMP, + FLOAT, + DOUBLE, + TEXT, + BLOB, + STRING, + OBJECT -> + (column, rowIndex, value) -> type.addValue(rowIndex, value, column); + case ROW, UNKNOWN, VECTOR -> + (column, rowIndex, value) -> { + throw new UnsupportedOperationException( + String.format( + DataNodePipeMessages.UNSUPPORTED_OUTPUT_DATATYPE_FMT, + type.getTypeEnum())); + }; + }; + + public static final TypeService + OPC_UA_TABLET_OBJECT_VALUE_GETTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (column, rowIndex) -> ((boolean[]) column)[rowIndex]; + case INT32 -> (column, rowIndex) -> ((int[]) column)[rowIndex]; + case DATE -> + (column, rowIndex) -> + new DateTime(Date.valueOf(((LocalDate[]) column)[rowIndex])); + case INT64 -> (column, rowIndex) -> ((long[]) column)[rowIndex]; + case TIMESTAMP -> + (column, rowIndex) -> + new DateTime( + TimestampPrecisionUtils.currPrecision.toNanos( + ((long[]) column)[rowIndex]) + / 100L + + 116444736000000000L); + case FLOAT -> (column, rowIndex) -> ((float[]) column)[rowIndex]; + case DOUBLE -> (column, rowIndex) -> ((double[]) column)[rowIndex]; + case TEXT, BLOB, STRING -> + (column, rowIndex) -> ((Binary[]) column)[rowIndex].toString(); + case OBJECT, ROW, UNKNOWN, VECTOR -> + (column, rowIndex) -> { + throw new UnSupportedDataTypeException( + DataNodePipeMessages.UNSUPPORTED_DATATYPE + type.getTypeEnum()); + }; + }; + + public static final TypeService> OPC_UA_DATA_TYPE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> () -> Identifiers.Boolean; + case INT32 -> () -> Identifiers.Int32; + case DATE, TIMESTAMP -> () -> Identifiers.DateTime; + case INT64 -> () -> Identifiers.Int64; + case FLOAT -> () -> Identifiers.Float; + case DOUBLE -> () -> Identifiers.Double; + case TEXT, BLOB, STRING -> () -> Identifiers.String; + case OBJECT, ROW, UNKNOWN, VECTOR -> + () -> { + throw new PipeRuntimeNonCriticalException( + DataNodePipeMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()); + }; + }; + + @FunctionalInterface + public interface TabletObjectValueGetter { + Object get(Object column, int rowIndex); + } + + @FunctionalInterface + public interface PipeRowObjectGetter { + Object get(Row row, int columnIndex); + } + + @FunctionalInterface + public interface AggregateRowValueUpdater { + Pair, Pair> update( + TimeSeriesRuntimeState state, + long timestamp, + Row row, + int columnIndex, + long reportInterval) + throws IOException; + } + + @FunctionalInterface + public interface AggregateTabletColumnValueWriter { + void write(Object column, int rowIndex, Object value); + } + + public static final TypeService> + PIPE_INSERT_EVENT_VALUE_LIST_TYPE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, + INT32, + INT64, + TIMESTAMP, + FLOAT, + DOUBLE, + TEXT, + BLOB, + OBJECT, + STRING -> + ignored -> type; + case DATE -> + isDateStoredAsLocalDate -> + isDateStoredAsLocalDate ? type : Type.fromTsDataType(TSDataType.INT32); + case ROW, UNKNOWN, VECTOR -> + ignored -> { + throw new UnSupportedDataTypeException( + DataNodePipeMessages.UNSUPPORTED_DATA_TYPE + type.getTypeEnum()) + .setChecked(true); + }; + }; + + public static final TypeService + PIPE_DATA_TYPE_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> org.apache.iotdb.pipe.api.type.Type.BOOLEAN; + case INT32 -> org.apache.iotdb.pipe.api.type.Type.INT32; + case INT64 -> org.apache.iotdb.pipe.api.type.Type.INT64; + case FLOAT -> org.apache.iotdb.pipe.api.type.Type.FLOAT; + case DOUBLE -> org.apache.iotdb.pipe.api.type.Type.DOUBLE; + case TEXT -> org.apache.iotdb.pipe.api.type.Type.TEXT; + case TIMESTAMP -> org.apache.iotdb.pipe.api.type.Type.TIMESTAMP; + case DATE -> org.apache.iotdb.pipe.api.type.Type.DATE; + case BLOB -> org.apache.iotdb.pipe.api.type.Type.BLOB; + case STRING -> org.apache.iotdb.pipe.api.type.Type.STRING; + case OBJECT -> + throw new IllegalArgumentException( + DataNodePipeMessages.INVALID_INPUT + TSDataType.OBJECT.getType()); + case UNKNOWN -> + throw new IllegalArgumentException( + DataNodePipeMessages.INVALID_INPUT + TSDataType.UNKNOWN.getType()); + case VECTOR -> + throw new IllegalArgumentException( + DataNodePipeMessages.INVALID_INPUT + TSDataType.VECTOR.getType()); + case ROW -> + throw new IllegalArgumentException( + DataNodePipeMessages.INVALID_INPUT + type.getTypeEnum()); + }; + + public static final TypeService PIPE_ROW_OBJECT_GETTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> Row::getBoolean; + case INT32 -> Row::getInt; + case DATE -> Row::getDate; + case INT64, TIMESTAMP -> Row::getLong; + case FLOAT -> Row::getFloat; + case DOUBLE -> Row::getDouble; + case TEXT, BLOB, STRING -> Row::getBinary; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (row, columnIndex) -> { + throw new UnsupportedOperationException( + String.format( + DataNodePipeMessages + .EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_FOR_COLUMN_ARG_4C4CCA6D, + row.getDataType(columnIndex), + row.getColumnName(columnIndex))); + }; + }; + + public static final TypeService AGGREGATE_ROW_VALUE_UPDATER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (state, timestamp, row, columnIndex, reportInterval) -> + state.updateWindows(timestamp, row.getBoolean(columnIndex), reportInterval); + case INT32 -> + (state, timestamp, row, columnIndex, reportInterval) -> + state.updateWindows(timestamp, row.getInt(columnIndex), reportInterval); + case DATE -> + (state, timestamp, row, columnIndex, reportInterval) -> + state.updateWindows(timestamp, row.getDate(columnIndex), reportInterval); + case INT64, TIMESTAMP -> + (state, timestamp, row, columnIndex, reportInterval) -> + state.updateWindows(timestamp, row.getLong(columnIndex), reportInterval); + case FLOAT -> + (state, timestamp, row, columnIndex, reportInterval) -> + state.updateWindows(timestamp, row.getFloat(columnIndex), reportInterval); + case DOUBLE -> + (state, timestamp, row, columnIndex, reportInterval) -> + state.updateWindows(timestamp, row.getDouble(columnIndex), reportInterval); + case TEXT, STRING -> + (state, timestamp, row, columnIndex, reportInterval) -> + state.updateWindows(timestamp, row.getString(columnIndex), reportInterval); + case BLOB, OBJECT -> + (state, timestamp, row, columnIndex, reportInterval) -> + state.updateWindows(timestamp, row.getBinary(columnIndex), reportInterval); + case ROW, UNKNOWN, VECTOR -> + (state, timestamp, row, columnIndex, reportInterval) -> { + throw new UnsupportedOperationException( + String.format( + DataNodePipeMessages + .EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_FOR_COLUMN_ARG_4C4CCA6D, + row.getDataType(columnIndex), + row.getColumnName(columnIndex))); + }; + }; + + public static final TypeService + PIPE_TS_PRIMITIVE_TABLET_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, DATE, INT64, TIMESTAMP, FLOAT, DOUBLE -> type::write; + case TEXT, BLOB, STRING -> + (value, column, index) -> { + type.write(value, column, index); + final Binary[] binaryColumn = (Binary[]) column; + binaryColumn[index] = normalizeBinaryValue(binaryColumn[index]); + }; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (value, column, index) -> { + throw new UnSupportedDataTypeException( + DataNodePipeMessages.UNSUPPORTED + type.getTypeEnum()); + }; + }; + + public static final TypeService + PIPE_BATCH_DATA_TABLET_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, DATE, INT64, TIMESTAMP, FLOAT, DOUBLE -> type::write; + case TEXT, BLOB, STRING -> + (value, column, index) -> { + type.write(value, column, index); + final Binary[] binaryColumn = (Binary[]) column; + binaryColumn[index] = normalizeBinaryValue(binaryColumn[index]); + }; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (value, column, index) -> { + throw new UnSupportedDataTypeException( + DataNodePipeMessages.UNSUPPORTED + type.getTypeEnum()); + }; + }; + + private static Binary normalizeBinaryValue(final Binary value) { + return Objects.isNull(value) || Objects.isNull(value.getValues()) + ? Binary.EMPTY_VALUE + : value; + } + + public static final TypeService + PIPE_TABLET_VALUE_COLUMN_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN, INT32, INT64, TIMESTAMP, FLOAT, DOUBLE -> + (originValueColumn, + rowIndexList, + isSingleOriginValueColumn, + originNullValueColumnBitmap, + nullValueColumnBitmap) -> + filterPrimitiveValueColumn( + type, + originValueColumn, + rowIndexList, + isSingleOriginValueColumn, + originNullValueColumnBitmap, + nullValueColumnBitmap); + case DATE -> Pipe::filterDateValueColumn; + case TEXT, BLOB, STRING -> Pipe::filterBinaryValueColumn; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (originValueColumn, + rowIndexList, + isSingleOriginValueColumn, + originNullValueColumnBitmap, + nullValueColumnBitmap) -> { + throw new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", type.getTypeEnum())); + }; + }; + + private static Object filterPrimitiveValueColumn( + final Type type, + final Object originValueColumn, + final List rowIndexList, + final boolean isSingleOriginValueColumn, + final BitMap originNullValueColumnBitmap, + final BitMap nullValueColumnBitmap) { + final Object originValueColumns; + if (isSingleOriginValueColumn) { + originValueColumns = type.createArray(1); + type.addValue(0, originValueColumn, originValueColumns); + } else { + originValueColumns = originValueColumn; + } + final Object valueColumns = type.createArray(rowIndexList.size()); + for (int i = 0; i < rowIndexList.size(); ++i) { + final int originRowIndex = rowIndexList.get(i); + if (isNullValue(originNullValueColumnBitmap, originRowIndex)) { + nullValueColumnBitmap.mark(i); + } else { + type.copyArrayElement(originValueColumns, originRowIndex, valueColumns, i); + } + } + return valueColumns; + } + + private static LocalDate[] filterDateValueColumn( + final Object originValueColumn, + final List rowIndexList, + final boolean isSingleOriginValueColumn, + final BitMap originNullValueColumnBitmap, + final BitMap nullValueColumnBitmap) { + final LocalDate[] valueColumns = new LocalDate[rowIndexList.size()]; + final boolean isLocalDateColumn = + isSingleOriginValueColumn + ? originValueColumn instanceof LocalDate + : originValueColumn instanceof LocalDate[]; + final LocalDate[] dateValueColumns = + isLocalDateColumn + ? (isSingleOriginValueColumn + ? new LocalDate[] {(LocalDate) originValueColumn} + : (LocalDate[]) originValueColumn) + : null; + final int[] intValueColumns = + isLocalDateColumn + ? null + : (isSingleOriginValueColumn + ? new int[] {(int) originValueColumn} + : (int[]) originValueColumn); + for (int i = 0; i < rowIndexList.size(); ++i) { + final int originRowIndex = rowIndexList.get(i); + if (isNullValue(originNullValueColumnBitmap, originRowIndex)) { + valueColumns[i] = EMPTY_LOCAL_DATE; + nullValueColumnBitmap.mark(i); + } else { + valueColumns[i] = + isLocalDateColumn + ? dateValueColumns[originRowIndex] + : DateUtils.parseIntToLocalDate(intValueColumns[originRowIndex]); + } + } + return valueColumns; + } + + private static Binary[] filterBinaryValueColumn( + final Object originValueColumn, + final List rowIndexList, + final boolean isSingleOriginValueColumn, + final BitMap originNullValueColumnBitmap, + final BitMap nullValueColumnBitmap) { + final Binary[] binaryValueColumns = + isSingleOriginValueColumn + ? new Binary[] {(Binary) originValueColumn} + : (Binary[]) originValueColumn; + final Binary[] valueColumns = new Binary[rowIndexList.size()]; + for (int i = 0; i < rowIndexList.size(); ++i) { + final int originRowIndex = rowIndexList.get(i); + final Binary value = binaryValueColumns[originRowIndex]; + if (Objects.isNull(value) + || Objects.isNull(value.getValues()) + || isNullValue(originNullValueColumnBitmap, originRowIndex)) { + valueColumns[i] = Binary.EMPTY_VALUE; + nullValueColumnBitmap.mark(i); + } else { + valueColumns[i] = new Binary(value.getValues()); + } + } + return valueColumns; + } + + private static boolean isNullValue(final BitMap bitMap, final int rowIndex) { + return Objects.nonNull(bitMap) && bitMap.isMarked(rowIndex); + } + + static { + SAME_TYPE_NUMERIC_OPERATOR_STRATEGY_SERVICE.check(); + OPC_DA_TABLET_VALUE_SETTER_SERVICE.check(); + OPC_UA_VALUE_STRINGIFIER_SERVICE.check(); + OPC_UA_LAST_VALUE_CONVERTER_SERVICE.check(); + CUSTOMIZED_INTERMEDIATE_RESULT_TO_INT_SERVICE.check(); + CUSTOMIZED_INTERMEDIATE_RESULT_TO_LONG_SERVICE.check(); + CUSTOMIZED_INTERMEDIATE_RESULT_TO_FLOAT_SERVICE.check(); + CUSTOMIZED_INTERMEDIATE_RESULT_TO_DOUBLE_SERVICE.check(); + CUSTOMIZED_INTERMEDIATE_RESULT_TO_STRING_SERVICE.check(); + AGGREGATE_TABLET_COLUMN_ALLOCATOR_SERVICE.check(); + AGGREGATE_TABLET_COLUMN_VALUE_WRITER_SERVICE.check(); + OPC_UA_TABLET_OBJECT_VALUE_GETTER_SERVICE.check(); + OPC_UA_DATA_TYPE_SERVICE.check(); + PIPE_INSERT_EVENT_VALUE_LIST_TYPE_SERVICE.check(); + // PIPE_DATA_TYPE_TRANSFORMER_SERVICE returns a value directly and intentionally rejects + // internal TsFile types, so the generic service check cannot be applied to it. + PIPE_ROW_OBJECT_GETTER_SERVICE.check(); + AGGREGATE_ROW_VALUE_UPDATER_SERVICE.check(); + PIPE_TS_PRIMITIVE_TABLET_VALUE_WRITER_SERVICE.check(); + PIPE_BATCH_DATA_TABLET_VALUE_WRITER_SERVICE.check(); + PIPE_TABLET_VALUE_COLUMN_FILTER_SERVICE.check(); + } + + private Pipe() { + // Utility class + } + } + + public static int parseInteger(final String value) { + try { + return Integer.parseInt(value); + } catch (final Exception e) { + return 0; + } + } + + public static long parseLong(final String value) { + try { + return Long.parseLong(value); + } catch (final Exception e) { + return 0L; + } + } + + public static float parseFloat(final String value) { + try { + return Float.parseFloat(value); + } catch (final Exception e) { + return 0.0f; + } + } + + public static double parseDouble(final String value) { + try { + return Double.parseDouble(value); + } catch (final Exception e) { + return 0.0d; + } + } + + public static Binary parseBlob(final String value) { + return new Binary(value, TSFileConfig.STRING_CHARSET); + } + + public static Binary parseString(final String value) { + return new Binary(value, TSFileConfig.STRING_CHARSET); + } + + public static Binary parseText(final String value) { + return new Binary(value, TSFileConfig.STRING_CHARSET); + } + + public static long parseTimestamp(final String value) { + if (value == null || value.isEmpty()) { + return 0L; + } + try { + return TypeInferenceUtils.isNumber(value) + ? Long.parseLong(value) + : DataNodeDateTimeUtils.parseDateTimeExpressionToLong( + StringUtils.trim(value), ZoneOffset.UTC); + } catch (final Exception e) { + return 0L; + } + } + + public static int parseDate(final String value) { + if (value == null) { + return DEFAULT_DATE; + } + final String trimmedValue = StringUtils.trim(value); + if (trimmedValue.isEmpty()) { + return DEFAULT_DATE; + } + if (TypeInferenceUtils.isNumber(trimmedValue)) { + try { + int date = Integer.parseInt(trimmedValue); + DateUtils.parseIntToLocalDate(date); + return date; + } catch (final Exception e) { + return DEFAULT_DATE; + } + } + try { + return DateTimeUtils.parseDateExpressionToInt(trimmedValue); + } catch (final Exception e) { + return parseDateTimeToDate(trimmedValue); + } + } + + public static int parseDateTimeToDate(final String value) { + try { + return DateUtils.parseDateExpressionToInt( + Instant.ofEpochMilli( + DateTimeUtils.convertDatetimeStrToLong(value, ZoneOffset.UTC, 0, "ms")) + .atZone(ZoneOffset.UTC) + .toLocalDate()); + } catch (final Exception e) { + return DEFAULT_DATE; + } + } + + private static long getLongValue(final Literal value) { + return ((LongLiteral) value).getParsedValue(); + } + + private static double getDoubleValue(final Literal value) { + if (value instanceof DoubleLiteral) { + return ((DoubleLiteral) value).getValue(); + } else if (value instanceof LongLiteral) { + return ((LongLiteral) value).getParsedValue(); + } else if (value instanceof FloatLiteral) { + return ((FloatLiteral) value).getValue(); + } + throw new IllegalArgumentException( + DataNodeQueryMessages.EXPRESSION_SHOULD_BE_NUMERIC_ACTUAL_IS + value); + } + + private static Long getTimestampValue(final Literal value) { + if (value instanceof LongLiteral) { + return ((LongLiteral) value).getParsedValue(); + } else if (value instanceof DoubleLiteral) { + return (long) ((DoubleLiteral) value).getValue(); + } else if (value instanceof GenericLiteral) { + return Long.valueOf(((GenericLiteral) value).getValue()); + } + throw new SemanticException( + String.format( + DataNodeQueryMessages + .EXCEPTION_TIMESTAMP_IN_LIST_LITERAL_CAN_ONLY_BE_LONGLITERAL_DOUBLELITERAL_OR_GENERICLITERAL_ACTUAL_TYPE_ARG_D751BE79, + value.getClass().getSimpleName())); + } + + @FunctionalInterface + public interface TabletValueColumnFilter { + Object filter( + Object originValueColumn, + List rowIndexList, + boolean isSingleOriginValueColumn, + BitMap originNullValueColumnBitmap, + BitMap nullValueColumnBitmap); + } + + @FunctionalInterface + public interface TsPrimitiveTabletValueWriter { + void write(TsPrimitiveType value, Object column, int index); + } + + @FunctionalInterface + public interface BatchDataTabletValueWriter { + void write(BatchData value, Object column, int index); + } + + @FunctionalInterface + public interface WALColumnWriter { + void write(Object column, IWALByteBufferView buffer, int start, int end); + } + + @FunctionalInterface + public interface TVListArrayWriter { + void write(TVList tvList, long[] times, Object values, BitMap bitMap, int start, int end); + } + + @FunctionalInterface + public interface TVListObjectWriter { + void write(TVList tvList, long time, Object value); + } + + public static class TVListProvider { + private final Supplier factory; + private final TVListDeserializer deserializer; + private final TVListDeserializer noBitmapDeserializer; + + private TVListProvider( + Supplier factory, + TVListDeserializer deserializer, + TVListDeserializer noBitmapDeserializer) { + this.factory = factory; + this.deserializer = deserializer; + this.noBitmapDeserializer = noBitmapDeserializer; + } + + public TVList newList() { + return factory.get(); + } + + public TVList deserialize(DataInputStream stream) throws IOException { + return deserializer.deserialize(stream); + } + + public TVList deserializeWithoutBitMap(DataInputStream stream) throws IOException { + return noBitmapDeserializer.deserialize(stream); + } + } + + @FunctionalInterface + private interface TVListDeserializer { + TVList deserialize(DataInputStream stream) throws IOException; + } + + @FunctionalInterface + public interface TVListChunkWriter { + long write(ChunkWriterImpl writer, long time, TVList tvList, int index); + } + + @FunctionalInterface + public interface TVListBatchWriter { + boolean write( + TVList tvList, + int index, + long time, + Filter filter, + TsBlockBuilder builder, + int floatPrecision, + TSEncoding encoding, + PaginationController paginationController); + } + + @FunctionalInterface + public interface AlignedTVListChunkWriter { + void write( + ValueChunkWriter writer, + long time, + AlignedTVList tvList, + int rowIndex, + int columnIndex, + boolean isNull); + } + + @FunctionalInterface + public interface ValueChunkArrayWriter { + void write(ValueChunkWriter writer, long time, Object values, int index, boolean isNull); + } + + @FunctionalInterface + public interface ValueChunkArrayBatchWriter { + void write( + ValueChunkWriter writer, + long[] timestamps, + Object values, + boolean[] nulls, + int arrayOffset, + int pointsInSegment); + } + + @FunctionalInterface + public interface ValueChunkArrayBitmapWriter { + void write( + ValueChunkWriter writer, + long[] timestamps, + Object values, + BitMap bitMap, + int arrayOffset, + int pointsInSegment); + } + + @FunctionalInterface + public interface AlignedTVListColumnWriter { + void write( + AlignedChunkWriterImpl writer, + long time, + AlignedTVList list, + int rowIndex, + int columnIndex, + boolean isNull); + } + + @FunctionalInterface + public interface SerializedColumnSizeCalculator { + int size(Object column, int start, int end); + + default int size(Object column, int rowCount) { + return size(column, 0, rowCount); + } + } + + @FunctionalInterface + public interface RawArrayByteBufferSerializer { + void serialize(Object column, int rowCount, ByteBuffer buffer); + } + + @FunctionalInterface + public interface RawArrayOutputStreamSerializer { + void serialize(Object column, int rowCount, DataOutputStream stream) throws IOException; + } + + @FunctionalInterface + public interface ArrayValueGetter { + Object get(Object array, int index); + } + + @FunctionalInterface + public interface SourceColumnToTabletValueWriter { + void write( + Type sourceType, + Column sourceColumn, + int sourceIndex, + Object targetColumn, + int targetIndex); + } + + @FunctionalInterface + public interface DecodedValueChunkWriter { + void write(ValueChunkWriter writer, long time, InputStream stream, boolean isNull) + throws IOException; + } + + @FunctionalInterface + public interface SegmentedArraySerializedSizeCalculator { + int calculate(List valueArrays, int rowCount, int arraySize); + } + + @FunctionalInterface + public interface ArrayValueColumnWriter { + void write( + ColumnBuilder builder, Object values, int index, int floatPrecision, TSEncoding encoding); + } + + @FunctionalInterface + public interface BatchDataColumnWriter { + void write(BatchData batchData, long time, Column column, int index); + } + + @FunctionalInterface + public interface ValueSerializer { + int serialize(T value, DataOutputStream stream) throws IOException; + } + + @FunctionalInterface + public interface TabletColumnDecoder { + Object decode(Decoder decoder, ByteBuffer buffer, int rowCount, TSEncoding encoding); + } + + @FunctionalInterface + public interface DecodedArrayValueReader { + void read(Object values, int index, InputStream stream) throws IOException; + } + + @FunctionalInterface + public interface RawArrayInputStreamDeserializer { + Object deserialize(DataInputStream stream, int size) throws IOException; + } + + @FunctionalInterface + public interface WindowValueArrayBuilder { + void build(WindowImpl window, EvictableBatchList list, int begin, int size); + } + + @FunctionalInterface + public interface DecodedChunkWriter { + void write(ChunkWriterImpl writer, long time, InputStream stream) throws IOException; + } + + @FunctionalInterface + public interface ChunkMetadataStatisticsConverter { + Statistics convert( + IChunkMetadata chunkMetadata, TSDataType targetDataType, Statistics statistics); + } + + @FunctionalInterface + public interface OpcDaTabletValueSetter { + WTypes.BSTR set(Variant.VARIANT value, Object column, int rowIndex); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index d2176c6891e4..495507d633d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -24,13 +24,12 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; -import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager; -import org.apache.iotdb.db.utils.MathUtils; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.block.column.Column; @@ -42,8 +41,8 @@ import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.TsBlockUtil; -import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; @@ -59,6 +58,7 @@ import java.io.DataInputStream; import java.io.IOException; +import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -479,34 +479,11 @@ public synchronized void putAlignedValue(long timestamp, Object[] value) { markNullValue(i, arrayIndex, elementIndex); continue; } + TSDataType dataType = dataTypes.get(i); Object valueArray = getOrCreateValueArray(i, arrayIndex, elementIndex); - switch (dataTypes.get(i)) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - ((Binary[]) valueArray)[elementIndex] = (Binary) columnValue; - memoryBinaryChunkSize[i] += getBinarySize((Binary) columnValue); - break; - case FLOAT: - ((float[]) valueArray)[elementIndex] = (float) columnValue; - break; - case INT32: - case DATE: - ((int[]) valueArray)[elementIndex] = (int) columnValue; - break; - case INT64: - case TIMESTAMP: - ((long[]) valueArray)[elementIndex] = (long) columnValue; - break; - case DOUBLE: - ((double[]) valueArray)[elementIndex] = (double) columnValue; - break; - case BOOLEAN: - ((boolean[]) valueArray)[elementIndex] = (boolean) columnValue; - break; - default: - break; + Type.fromTsDataType(dataType).addValue(elementIndex, columnValue, valueArray); + if (dataType.isBinary()) { + memoryBinaryChunkSize[i] += getBinarySize((Binary) columnValue); } } if (indices != null) { @@ -568,65 +545,70 @@ private TsPrimitiveType getAlignedValueByValueIndex( } else { validValueIndex = valueIndex; } - int arrayIndex = validValueIndex / ARRAY_SIZE; - int elementIndex = validValueIndex % ARRAY_SIZE; if (columnValues == null || isNullValue(validValueIndex, columnIndex)) { continue; } - switch (dataTypes.get(columnIndex)) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary valueT = ((Binary[]) columnValues.get(arrayIndex))[elementIndex]; - vector[columnIndex] = TsPrimitiveType.getByType(TSDataType.TEXT, valueT); - break; - case FLOAT: - float valueF = ((float[]) columnValues.get(arrayIndex))[elementIndex]; - if (floatPrecision != null - && encodingList != null - && !Float.isNaN(valueF) - && (encodingList.get(columnIndex) == TSEncoding.RLE - || encodingList.get(columnIndex) == TSEncoding.TS_2DIFF)) { - valueF = MathUtils.roundWithGivenPrecision(valueF, floatPrecision); - } - vector[columnIndex] = TsPrimitiveType.getByType(TSDataType.FLOAT, valueF); - break; - case INT32: - case DATE: - int valueI = ((int[]) columnValues.get(arrayIndex))[elementIndex]; - vector[columnIndex] = TsPrimitiveType.getByType(TSDataType.INT32, valueI); - break; - case INT64: - case TIMESTAMP: - long valueL = ((long[]) columnValues.get(arrayIndex))[elementIndex]; - vector[columnIndex] = TsPrimitiveType.getByType(TSDataType.INT64, valueL); - break; - case DOUBLE: - double valueD = ((double[]) columnValues.get(arrayIndex))[elementIndex]; - if (floatPrecision != null - && encodingList != null - && !Double.isNaN(valueD) - && (encodingList.get(columnIndex) == TSEncoding.RLE - || encodingList.get(columnIndex) == TSEncoding.TS_2DIFF)) { - valueD = MathUtils.roundWithGivenPrecision(valueD, floatPrecision); - } - vector[columnIndex] = TsPrimitiveType.getByType(TSDataType.DOUBLE, valueD); - break; - case BOOLEAN: - boolean valueB = ((boolean[]) columnValues.get(arrayIndex))[elementIndex]; - vector[columnIndex] = TsPrimitiveType.getByType(TSDataType.BOOLEAN, valueB); - break; - default: - throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); + vector[columnIndex] = + getPrimitiveTypeByValueIndex( + validValueIndex, + columnIndex, + getPhysicalType(dataTypes.get(columnIndex)), + dataTypes.get(columnIndex), + dataTypes.get(columnIndex), + floatPrecision, + encodingList == null ? null : encodingList.get(columnIndex)); + } + return Type.fromTsDataType(TSDataType.VECTOR).getTsPrimitiveType(vector); + } + + private static Type getPhysicalType(TSDataType dataType) { + if (dataType == TSDataType.DATE) { + return Type.fromTsDataType(TSDataType.INT32); + } + if (dataType == TSDataType.TIMESTAMP) { + return Type.fromTsDataType(TSDataType.INT64); + } + if (dataType.isBinary()) { + return Type.fromTsDataType(TSDataType.TEXT); + } + return Type.fromTsDataType(dataType); + } + + private TsPrimitiveType getPrimitiveTypeByValueIndex( + int valueIndex, + int columnIndex, + Type resultType, + TSDataType sourceDataType, + TSDataType targetDataType, + Integer floatPrecision, + TSEncoding encoding) { + int arrayIndex = valueIndex / ARRAY_SIZE; + int elementIndex = valueIndex % ARRAY_SIZE; + TsPrimitiveType value; + if (sourceDataType == targetDataType) { + value = + resultType.getValueAsTsPrimitiveType( + values.get(columnIndex).get(arrayIndex), elementIndex); + } else { + value = + resultType.getTsPrimitiveType( + targetDataType.castFromSingleValue( + sourceDataType, getObjectByValueIndex(valueIndex, columnIndex))); + } + if (floatPrecision != null && encoding != null) { + if (value.getDataType() == TSDataType.FLOAT) { + value.setFloat(roundValueWithGivenPrecision(value.getFloat(), floatPrecision, encoding)); + } else if (value.getDataType() == TSDataType.DOUBLE) { + value.setDouble(roundValueWithGivenPrecision(value.getDouble(), floatPrecision, encoding)); } } - return TsPrimitiveType.getByType(TSDataType.VECTOR, vector); + return value; } public void extendColumn(TSDataType dataType) { List columnValue = new ArrayList<>(timestamps.size()); for (int i = 0; i < timestamps.size(); i++) { + // New columns contain only implicit nulls until their first non-null write. columnValue.add(null); } if (bitMaps != null) { @@ -647,28 +629,9 @@ private Object getObjectByValueIndex(int rowIndex, int columnIndex) { int arrayIndex = rowIndex / ARRAY_SIZE; int elementIndex = rowIndex % ARRAY_SIZE; List columnValues = values.get(columnIndex); - switch (dataTypes.get(columnIndex)) { - case INT32: - case DATE: - return ((int[]) columnValues.get(arrayIndex))[elementIndex]; - case INT64: - case TIMESTAMP: - return ((long[]) columnValues.get(arrayIndex))[elementIndex]; - case FLOAT: - return ((float[]) columnValues.get(arrayIndex))[elementIndex]; - case DOUBLE: - return ((double[]) columnValues.get(arrayIndex))[elementIndex]; - case BOOLEAN: - return ((boolean[]) columnValues.get(arrayIndex))[elementIndex]; - case STRING: - case BLOB: - case TEXT: - case OBJECT: - return ((Binary[]) columnValues.get(arrayIndex))[elementIndex]; - default: - throw new IllegalArgumentException( - dataTypes.get(columnIndex) + StorageEngineMessages.IS_NOT_SUPPORTED); - } + return TypeServices.StorageEngine.ARRAY_VALUE_GETTER_SERVICE + .call(Type.fromTsDataType(dataTypes.get(columnIndex))) + .get(columnValues.get(arrayIndex), elementIndex); } /** @@ -1007,45 +970,7 @@ protected Object cloneValue(TSDataType type, Object value) { if (value == null) { return null; } - switch (type) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary[] valueT = (Binary[]) value; - Binary[] cloneT = new Binary[valueT.length]; - System.arraycopy(valueT, 0, cloneT, 0, valueT.length); - return cloneT; - case FLOAT: - float[] valueF = (float[]) value; - float[] cloneF = new float[valueF.length]; - System.arraycopy(valueF, 0, cloneF, 0, valueF.length); - return cloneF; - case INT32: - case DATE: - int[] valueI = (int[]) value; - int[] cloneI = new int[valueI.length]; - System.arraycopy(valueI, 0, cloneI, 0, valueI.length); - return cloneI; - case INT64: - case TIMESTAMP: - long[] valueL = (long[]) value; - long[] cloneL = new long[valueL.length]; - System.arraycopy(valueL, 0, cloneL, 0, valueL.length); - return cloneL; - case DOUBLE: - double[] valueD = (double[]) value; - double[] cloneD = new double[valueD.length]; - System.arraycopy(valueD, 0, cloneD, 0, valueD.length); - return cloneD; - case BOOLEAN: - boolean[] valueB = (boolean[]) value; - boolean[] cloneB = new boolean[valueB.length]; - System.arraycopy(valueB, 0, cloneB, 0, valueB.length); - return cloneB; - default: - return null; - } + return Type.fromTsDataType(type).arrayCopyOf(value, Array.getLength(value)); } /* @@ -1354,45 +1279,18 @@ private void arrayCopy( if (value[i] == null || !containsNonNullValue(bitMaps, results, i, idx, remaining)) { continue; } - Object valueArray = getOrCreateValueArray(i, arrayIndex, elementIndex); - switch (dataTypes.get(i)) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary[] arrayT = (Binary[]) valueArray; - System.arraycopy(value[i], idx, arrayT, elementIndex, remaining); - - // update raw size of Text chunk - for (int i1 = 0; i1 < remaining; i1++) { - memoryBinaryChunkSize[i] += - arrayT[elementIndex + i1] != null ? getBinarySize(arrayT[elementIndex + i1]) : 0; - } - break; - case FLOAT: - float[] arrayF = (float[]) valueArray; - System.arraycopy(value[i], idx, arrayF, elementIndex, remaining); - break; - case INT32: - case DATE: - int[] arrayI = (int[]) valueArray; - System.arraycopy(value[i], idx, arrayI, elementIndex, remaining); - break; - case INT64: - case TIMESTAMP: - long[] arrayL = (long[]) valueArray; - System.arraycopy(value[i], idx, arrayL, elementIndex, remaining); - break; - case DOUBLE: - double[] arrayD = (double[]) valueArray; - System.arraycopy(value[i], idx, arrayD, elementIndex, remaining); - break; - case BOOLEAN: - boolean[] arrayB = (boolean[]) valueArray; - System.arraycopy(value[i], idx, arrayB, elementIndex, remaining); - break; - default: - break; + Object targetArray = getOrCreateValueArray(i, arrayIndex, elementIndex); + System.arraycopy(value[i], idx, targetArray, elementIndex, remaining); + + if (dataTypes.get(i).isBinary()) { + Binary[] binaryValues = (Binary[]) targetArray; + // update raw size of Text chunk + for (int valueIndex = 0; valueIndex < remaining; valueIndex++) { + memoryBinaryChunkSize[i] += + binaryValues[elementIndex + valueIndex] != null + ? getBinarySize(binaryValues[elementIndex + valueIndex]) + : 0; + } } } } @@ -1821,6 +1719,9 @@ public TsBlock buildTsBlock( lastValidPointIndexForTimeDupCheck = new Pair<>(Long.MIN_VALUE, null); } ColumnBuilder valueBuilder = builder.getColumnBuilder(columnIndex); + TypeServices.ArrayValueColumnWriter valueWriter = + TypeServices.StorageEngine.ARRAY_VALUE_COLUMN_WRITER_SERVICE.call( + Type.fromTsDataType(dataTypes.get(columnIndex))); currentWriteRowIndex = 0; for (int sortedRowIndex = 0; sortedRowIndex < rowCount; sortedRowIndex++) { // skip empty row @@ -1863,48 +1764,14 @@ public TsBlock buildTsBlock( continue; } hasAnyNonNullValue[currentWriteRowIndex++] = true; - switch (dataTypes.get(columnIndex)) { - case BOOLEAN: - valueBuilder.writeBoolean(getBooleanByValueIndex(originRowIndex, columnIndex)); - break; - case INT32: - valueBuilder.writeInt(getIntByValueIndex(originRowIndex, columnIndex)); - break; - case DATE: - if (valueBuilder instanceof BinaryColumnBuilder) { - ((BinaryColumnBuilder) valueBuilder) - .writeDate(getIntByValueIndex(originRowIndex, columnIndex)); - } else { - valueBuilder.writeInt(getIntByValueIndex(originRowIndex, columnIndex)); - } - break; - case INT64: - case TIMESTAMP: - valueBuilder.writeLong(getLongByValueIndex(originRowIndex, columnIndex)); - break; - case FLOAT: - valueBuilder.writeFloat( - roundValueWithGivenPrecision( - getFloatByValueIndex(originRowIndex, columnIndex), - floatPrecision, - encodingList.get(columnIndex))); - break; - case DOUBLE: - valueBuilder.writeDouble( - roundValueWithGivenPrecision( - getDoubleByValueIndex(originRowIndex, columnIndex), - floatPrecision, - encodingList.get(columnIndex))); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueBuilder.writeBinary(getBinaryByValueIndex(originRowIndex, columnIndex)); - break; - default: - break; - } + int arrayIndex = originRowIndex / ARRAY_SIZE; + int elementIndex = originRowIndex % ARRAY_SIZE; + valueWriter.write( + valueBuilder, + values.get(columnIndex).get(arrayIndex), + elementIndex, + floatPrecision, + encodingList.get(columnIndex)); } } builder.declarePositions(validRowCount); @@ -1973,41 +1840,11 @@ public int serializedSize() { size += rowCount * Long.BYTES; // value for (int columnIndex = 0; columnIndex < values.size(); ++columnIndex) { - switch (dataTypes.get(columnIndex)) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - for (int rowIdx = 0; rowIdx < rowCount; ++rowIdx) { - int arrayIndex = rowIdx / ARRAY_SIZE; - Object valueArray = values.get(columnIndex).get(arrayIndex); - Binary value = - valueArray == null - ? Binary.EMPTY_VALUE - : ((Binary[]) valueArray)[rowIdx % ARRAY_SIZE]; - size += ReadWriteIOUtils.sizeToWrite(value == null ? Binary.EMPTY_VALUE : value); - } - break; - case FLOAT: - size += rowCount * Float.BYTES; - break; - case INT32: - case DATE: - size += rowCount * Integer.BYTES; - break; - case INT64: - case TIMESTAMP: - size += rowCount * Long.BYTES; - break; - case DOUBLE: - size += rowCount * Double.BYTES; - break; - case BOOLEAN: - size += rowCount * Byte.BYTES; - break; - default: - throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); - } + Type type = Type.fromTsDataType(dataTypes.get(columnIndex)); + size += + TypeServices.StorageEngine.SEGMENTED_ARRAY_SERIALIZED_SIZE_SERVICE + .call(type) + .calculate(values.get(columnIndex), rowCount, ARRAY_SIZE); } // bitmap size += rowCount * dataTypes.size() * Byte.BYTES; @@ -2035,56 +1872,16 @@ public void serializeToWAL(IWALByteBufferView buffer) { } // serialize value and bitmap by column for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) { + TypeServices.WALColumnWriter valueWriter = + TypeServices.StorageEngine.WAL_ARRAY_WRITER_SERVICE.call( + Type.fromTsDataType(dataTypes.get(columnIndex))); List columnValues = values.get(columnIndex); for (int rowIndex = 0; rowIndex < rowCount; ++rowIndex) { int arrayIndex = rowIndex / ARRAY_SIZE; int elementIndex = rowIndex % ARRAY_SIZE; Object valueArray = columnValues.get(arrayIndex); // value - switch (dataTypes.get(columnIndex)) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary valueT = - valueArray == null ? Binary.EMPTY_VALUE : ((Binary[]) valueArray)[elementIndex]; - // In some scenario, the Binary in AlignedTVList will be null if this field is empty in - // current row. We need to handle this scenario to get rid of NPE. See the similar issue - // here: https://github.com/apache/iotdb/pull/9884 - // Furthermore, we use an empty Binary as a placeholder here. It won't lead to data - // error because whether this field is null or not is decided by the bitMap rather than - // the object's value here. - if (valueT != null) { - WALWriteUtils.write(valueT, buffer); - } else { - WALWriteUtils.write(Binary.EMPTY_VALUE, buffer); - } - break; - case FLOAT: - float valueF = valueArray == null ? 0 : ((float[]) valueArray)[elementIndex]; - buffer.putFloat(valueF); - break; - case INT32: - case DATE: - int valueI = valueArray == null ? 0 : ((int[]) valueArray)[elementIndex]; - buffer.putInt(valueI); - break; - case INT64: - case TIMESTAMP: - long valueL = valueArray == null ? 0 : ((long[]) valueArray)[elementIndex]; - buffer.putLong(valueL); - break; - case DOUBLE: - double valueD = valueArray == null ? 0 : ((double[]) valueArray)[elementIndex]; - buffer.putDouble(valueD); - break; - case BOOLEAN: - boolean valueB = valueArray != null && ((boolean[]) valueArray)[elementIndex]; - WALWriteUtils.write(valueB, buffer); - break; - default: - throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); - } + valueWriter.write(valueArray, buffer, elementIndex, elementIndex + 1); // bitmap WALWriteUtils.write(isNullValue(rowIndex, columnIndex), buffer); } @@ -2121,76 +1918,17 @@ public static AlignedTVList deserialize(DataInputStream stream) throws IOExcepti Object[] values = new Object[dataTypeNum]; BitMap[] bitMaps = new BitMap[dataTypeNum]; for (int columnIndex = 0; columnIndex < dataTypeNum; ++columnIndex) { - BitMap bitMap = BitMap.createBitMapDynamically(rowCount); - Object valuesOfOneColumn; - switch (dataTypes.get(columnIndex)) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary[] binaryValues = new Binary[rowCount]; - for (int rowIndex = 0; rowIndex < rowCount; ++rowIndex) { - binaryValues[rowIndex] = ReadWriteIOUtils.readBinary(stream); - if (ReadWriteIOUtils.readBool(stream)) { - bitMap.mark(rowIndex); - } - } - valuesOfOneColumn = binaryValues; - break; - case FLOAT: - float[] floatValues = new float[rowCount]; - for (int rowIndex = 0; rowIndex < rowCount; ++rowIndex) { - floatValues[rowIndex] = stream.readFloat(); - if (ReadWriteIOUtils.readBool(stream)) { - bitMap.mark(rowIndex); - } - } - valuesOfOneColumn = floatValues; - break; - case INT32: - case DATE: - int[] intValues = new int[rowCount]; - for (int rowIndex = 0; rowIndex < rowCount; ++rowIndex) { - intValues[rowIndex] = stream.readInt(); - if (ReadWriteIOUtils.readBool(stream)) { - bitMap.mark(rowIndex); - } - } - valuesOfOneColumn = intValues; - break; - case INT64: - case TIMESTAMP: - long[] longValues = new long[rowCount]; - for (int rowIndex = 0; rowIndex < rowCount; ++rowIndex) { - longValues[rowIndex] = stream.readLong(); - if (ReadWriteIOUtils.readBool(stream)) { - bitMap.mark(rowIndex); - } - } - valuesOfOneColumn = longValues; - break; - case DOUBLE: - double[] doubleValues = new double[rowCount]; - for (int rowIndex = 0; rowIndex < rowCount; ++rowIndex) { - doubleValues[rowIndex] = stream.readDouble(); - if (ReadWriteIOUtils.readBool(stream)) { - bitMap.mark(rowIndex); - } - } - valuesOfOneColumn = doubleValues; - break; - case BOOLEAN: - boolean[] booleanValues = new boolean[rowCount]; - for (int rowIndex = 0; rowIndex < rowCount; ++rowIndex) { - booleanValues[rowIndex] = ReadWriteIOUtils.readBool(stream); - if (ReadWriteIOUtils.readBool(stream)) { - bitMap.mark(rowIndex); - } - } - valuesOfOneColumn = booleanValues; - break; - default: - throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); + BitMap bitMap = new BitMap(rowCount); + Type type = Type.fromTsDataType(dataTypes.get(columnIndex)); + Object valuesOfOneColumn = + TypeServices.StorageEngine.PRIMITIVE_ARRAY_ALLOCATOR_SERVICE.call(type).apply(rowCount); + TypeServices.DecodedArrayValueReader valueReader = + TypeServices.StorageEngine.DECODED_ARRAY_VALUE_READER_SERVICE.call(type); + for (int rowIndex = 0; rowIndex < rowCount; ++rowIndex) { + valueReader.read(valuesOfOneColumn, rowIndex, stream); + if (ReadWriteIOUtils.readBool(stream)) { + bitMap.mark(rowIndex); + } } values[columnIndex] = valuesOfOneColumn; bitMaps[columnIndex] = bitMap; @@ -2365,6 +2103,7 @@ public class AlignedTVListIterator extends TVListIterator { private final int floatPrecision; private final List encodingList; private final boolean ignoreAllNullRows; + private final List chunkValueWriters; // remember the selected index of last not-null value for each column during prepareNext phase private final int[] selectedIndices; @@ -2406,6 +2145,13 @@ public AlignedTVListIterator( this.timeColumnDeletion = timeColumnDeletion; this.valueColumnsDeletionList = valueColumnsDeletionList; this.ignoreAllNullRows = ignoreAllNullRows; + this.chunkValueWriters = + dataTypeList.stream() + .map( + dataType -> + TypeServices.StorageEngine.ALIGNED_TV_LIST_CHUNK_WRITER_SERVICE.call( + Type.fromTsDataType(dataType))) + .collect(Collectors.toList()); this.selectedIndices = new int[dataTypeList.size()]; timeDeleteCursor[0] = (timeColumnDeletion == null || scanOrder.isAscending()) @@ -2544,7 +2290,7 @@ public TimeValuePair nextTimeValuePair() { TimeValuePair tvPair = new TimeValuePair( getTime(getScanOrderIndex(index)), - TsPrimitiveType.getByType(TSDataType.VECTOR, vector)); + Type.fromTsDataType(TSDataType.VECTOR).getTsPrimitiveType(vector)); next(); return tvPair; @@ -2560,7 +2306,8 @@ public TimeValuePair currentTimeValuePair() { vector[columnIndex] = getPrimitiveTypeObject(selectedIndices[columnIndex], columnIndex); } return new TimeValuePair( - getTime(getScanOrderIndex(index)), TsPrimitiveType.getByType(TSDataType.VECTOR, vector)); + getTime(getScanOrderIndex(index)), + Type.fromTsDataType(TSDataType.VECTOR).getTsPrimitiveType(vector)); } public TsPrimitiveType getPrimitiveTypeObject(int rowIndex, int columnIndex) { @@ -2576,46 +2323,20 @@ public TsPrimitiveType getPrimitiveTypeObject(int rowIndex, int columnIndex) { if (outer.isNullValue(valueIndex, validColumnIndex)) { return null; } - switch (dataTypeList.get(columnIndex)) { - case BOOLEAN: - return TsPrimitiveType.getByType( - TSDataType.BOOLEAN, getBooleanByValueIndex(valueIndex, validColumnIndex)); - case INT32: - return TsPrimitiveType.getByType( - TSDataType.INT32, getIntByValueIndex(valueIndex, validColumnIndex)); - case DATE: - return TsPrimitiveType.getByType( - TSDataType.DATE, getIntByValueIndex(valueIndex, validColumnIndex)); - case INT64: - case TIMESTAMP: - return TsPrimitiveType.getByType( - TSDataType.INT64, getLongByValueIndex(valueIndex, validColumnIndex)); - case FLOAT: - float valueF = getFloatByValueIndex(valueIndex, validColumnIndex); - if (encodingList != null) { - valueF = - roundValueWithGivenPrecision(valueF, floatPrecision, encodingList.get(columnIndex)); - } - return TsPrimitiveType.getByType(TSDataType.FLOAT, valueF); - case DOUBLE: - double valueD = getDoubleByValueIndex(valueIndex, validColumnIndex); - if (encodingList != null) { - valueD = - roundValueWithGivenPrecision(valueD, floatPrecision, encodingList.get(columnIndex)); - } - return TsPrimitiveType.getByType(TSDataType.DOUBLE, valueD); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return TsPrimitiveType.getByType( - TSDataType.TEXT, getBinaryByValueIndex(valueIndex, validColumnIndex)); - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataTypeList.get(columnIndex))); - } + TSDataType sourceDataType = dataTypes.get(validColumnIndex); + TSDataType targetDataType = dataTypeList.get(columnIndex); + Type resultType = + targetDataType == TSDataType.DATE + ? Type.fromTsDataType(targetDataType) + : getPhysicalType(targetDataType); + return getPrimitiveTypeByValueIndex( + valueIndex, + validColumnIndex, + resultType, + sourceDataType, + targetDataType, + floatPrecision, + encodingList == null ? null : encodingList.get(columnIndex)); } @Override @@ -2737,6 +2458,11 @@ public TsBlock nextBatch() { lastValidPointIndexForTimeDupCheck = new Pair<>(Long.MIN_VALUE, null); } ColumnBuilder valueBuilder = builder.getColumnBuilder(columnIndex); + TypeServices.ArrayValueColumnWriter valueWriter = + validColumnIndex < 0 || validColumnIndex >= dataTypes.size() + ? null + : TypeServices.StorageEngine.ARRAY_VALUE_COLUMN_WRITER_SERVICE.call( + Type.fromTsDataType(dataTypes.get(validColumnIndex))); currentWriteRowIndex = 0; for (int sortedRowIndex = startIndex; sortedRowIndex < index; sortedRowIndex++) { // skip invalid rows @@ -2811,7 +2537,7 @@ public TsBlock nextBatch() { continue; } hasAnyNonNullValue[currentWriteRowIndex++] = true; - writeToColumn(validColumnIndex, valueBuilder, originRowIndex, columnIndex); + writeToColumn(validColumnIndex, valueBuilder, originRowIndex, columnIndex, valueWriter); } } builder.declarePositions(validRowCount); @@ -2848,51 +2574,40 @@ public TsBlock nextBatch() { } private void writeToColumn( - int validColumnIndex, ColumnBuilder valueBuilder, int originRowIndex, int columnIndex) { - switch (dataTypes.get(validColumnIndex)) { - case BOOLEAN: - valueBuilder.writeBoolean(getBooleanByValueIndex(originRowIndex, validColumnIndex)); - break; - case INT32: - valueBuilder.writeInt(getIntByValueIndex(originRowIndex, validColumnIndex)); - break; - case DATE: - if (valueBuilder instanceof BinaryColumnBuilder) { - ((BinaryColumnBuilder) valueBuilder) - .writeDate(getIntByValueIndex(originRowIndex, validColumnIndex)); - } else { - valueBuilder.writeInt(getIntByValueIndex(originRowIndex, validColumnIndex)); - } - break; - case INT64: - case TIMESTAMP: - valueBuilder.writeLong(getLongByValueIndex(originRowIndex, validColumnIndex)); - break; - case FLOAT: - float valueF = getFloatByValueIndex(originRowIndex, validColumnIndex); - if (encodingList != null) { - valueF = - roundValueWithGivenPrecision(valueF, floatPrecision, encodingList.get(columnIndex)); - } - valueBuilder.writeFloat(valueF); - break; - case DOUBLE: - double valueD = getDoubleByValueIndex(originRowIndex, validColumnIndex); - if (encodingList != null) { - valueD = - roundValueWithGivenPrecision(valueD, floatPrecision, encodingList.get(columnIndex)); - } - valueBuilder.writeDouble(valueD); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueBuilder.writeBinary(getBinaryByValueIndex(originRowIndex, validColumnIndex)); - break; - default: - break; + int validColumnIndex, + ColumnBuilder valueBuilder, + int originRowIndex, + int columnIndex, + TypeServices.ArrayValueColumnWriter valueWriter) { + int arrayIndex = originRowIndex / ARRAY_SIZE; + int elementIndex = originRowIndex % ARRAY_SIZE; + TSDataType sourceDataType = dataTypes.get(validColumnIndex); + TSDataType targetDataType = dataTypeList.get(columnIndex); + TSEncoding encoding = encodingList == null ? null : encodingList.get(columnIndex); + if (sourceDataType == targetDataType) { + valueWriter.write( + valueBuilder, + values.get(validColumnIndex).get(arrayIndex), + elementIndex, + floatPrecision, + encoding); + return; } + + Type resultType = + targetDataType == TSDataType.DATE + ? Type.fromTsDataType(targetDataType) + : getPhysicalType(targetDataType); + TsPrimitiveType value = + getPrimitiveTypeByValueIndex( + originRowIndex, + validColumnIndex, + resultType, + sourceDataType, + targetDataType, + floatPrecision, + encoding); + Type.fromTsDataType(targetDataType).write(valueBuilder, value); } private TsBlock reBuildTsBlock( @@ -3044,47 +2759,9 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo } boolean isNull = outer.isNullValue(originRowIndex, validColumnIndex); - switch (dataTypeList.get(columnIndex)) { - case BOOLEAN: - valueChunkWriter.write( - time, - !isNull && getBooleanByValueIndex(originRowIndex, validColumnIndex), - isNull); - break; - case INT32: - case DATE: - valueChunkWriter.write( - time, isNull ? 0 : getIntByValueIndex(originRowIndex, validColumnIndex), isNull); - break; - case INT64: - case TIMESTAMP: - valueChunkWriter.write( - time, isNull ? 0 : getLongByValueIndex(originRowIndex, validColumnIndex), isNull); - break; - case FLOAT: - valueChunkWriter.write( - time, - isNull ? 0 : getFloatByValueIndex(originRowIndex, validColumnIndex), - isNull); - break; - case DOUBLE: - valueChunkWriter.write( - time, - isNull ? 0 : getDoubleByValueIndex(originRowIndex, validColumnIndex), - isNull); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueChunkWriter.write( - time, - isNull ? null : getBinaryByValueIndex(originRowIndex, validColumnIndex), - isNull); - break; - default: - break; - } + chunkValueWriters + .get(columnIndex) + .write(valueChunkWriter, time, outer, originRowIndex, validColumnIndex, isNull); } } probeNext = false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java index b274bc8ef658..3583b99ccd8d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java @@ -27,10 +27,10 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataInputStream; import java.io.IOException; @@ -146,13 +146,14 @@ protected void expandValues() { @Override public TimeValuePair getTimeValuePair(int index) { return new TimeValuePair( - getTime(index), TsPrimitiveType.getByType(TSDataType.TEXT, getBinary(index))); + getTime(index), Type.fromTsDataType(TSDataType.TEXT).getTsPrimitiveType(getBinary(index))); } @Override protected TimeValuePair getTimeValuePair( int index, long time, Integer floatPrecision, TSEncoding encoding) { - return new TimeValuePair(time, TsPrimitiveType.getByType(TSDataType.TEXT, getBinary(index))); + return new TimeValuePair( + time, Type.fromTsDataType(TSDataType.TEXT).getTsPrimitiveType(getBinary(index))); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java index 5e3461acb2c8..b1cd21a01f18 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java @@ -27,9 +27,9 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataInputStream; import java.io.IOException; @@ -145,14 +145,15 @@ protected void expandValues() { @Override public TimeValuePair getTimeValuePair(int index) { return new TimeValuePair( - getTime(index), TsPrimitiveType.getByType(TSDataType.BOOLEAN, getBoolean(index))); + getTime(index), + Type.fromTsDataType(TSDataType.BOOLEAN).getTsPrimitiveType(getBoolean(index))); } @Override protected TimeValuePair getTimeValuePair( int index, long time, Integer floatPrecision, TSEncoding encoding) { return new TimeValuePair( - time, TsPrimitiveType.getByType(TSDataType.BOOLEAN, getBoolean(index))); + time, Type.fromTsDataType(TSDataType.BOOLEAN).getTsPrimitiveType(getBoolean(index))); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java index 753ca2020a89..a7e5d7dbae3c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java @@ -28,9 +28,9 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataInputStream; import java.io.IOException; @@ -146,7 +146,8 @@ protected void expandValues() { @Override public TimeValuePair getTimeValuePair(int index) { return new TimeValuePair( - getTime(index), TsPrimitiveType.getByType(TSDataType.DOUBLE, getDouble(index))); + getTime(index), + Type.fromTsDataType(TSDataType.DOUBLE).getTsPrimitiveType(getDouble(index))); } @Override @@ -156,7 +157,8 @@ protected TimeValuePair getTimeValuePair( if (!Double.isNaN(value) && (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF)) { value = MathUtils.roundWithGivenPrecision(value, floatPrecision); } - return new TimeValuePair(time, TsPrimitiveType.getByType(TSDataType.DOUBLE, value)); + return new TimeValuePair( + time, Type.fromTsDataType(TSDataType.DOUBLE).getTsPrimitiveType(value)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java index 517dc208211f..06566b8b9f1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java @@ -28,9 +28,9 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataInputStream; import java.io.IOException; @@ -146,7 +146,7 @@ protected void expandValues() { @Override public TimeValuePair getTimeValuePair(int index) { return new TimeValuePair( - getTime(index), TsPrimitiveType.getByType(TSDataType.FLOAT, getFloat(index))); + getTime(index), Type.fromTsDataType(TSDataType.FLOAT).getTsPrimitiveType(getFloat(index))); } @Override @@ -156,7 +156,7 @@ protected TimeValuePair getTimeValuePair( if (!Float.isNaN(value) && (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF)) { value = MathUtils.roundWithGivenPrecision(value, floatPrecision); } - return new TimeValuePair(time, TsPrimitiveType.getByType(TSDataType.FLOAT, value)); + return new TimeValuePair(time, Type.fromTsDataType(TSDataType.FLOAT).getTsPrimitiveType(value)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java index a51e785d414f..725ca7a2da76 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java @@ -27,9 +27,9 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataInputStream; import java.io.IOException; @@ -144,13 +144,14 @@ protected void expandValues() { @Override public TimeValuePair getTimeValuePair(int index) { - return new TimeValuePair(getTime(index), TsPrimitiveType.getByType(dataType, getInt(index))); + return new TimeValuePair( + getTime(index), Type.fromTsDataType(dataType).getTsPrimitiveType(getInt(index))); } @Override protected TimeValuePair getTimeValuePair( int index, long time, Integer floatPrecision, TSEncoding encoding) { - return new TimeValuePair(time, TsPrimitiveType.getByType(dataType, getInt(index))); + return new TimeValuePair(time, Type.fromTsDataType(dataType).getTsPrimitiveType(getInt(index))); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java index 544d2cb7dd0e..823142c9ba64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java @@ -27,9 +27,9 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataInputStream; import java.io.IOException; @@ -145,13 +145,14 @@ protected void expandValues() { @Override public TimeValuePair getTimeValuePair(int index) { return new TimeValuePair( - getTime(index), TsPrimitiveType.getByType(TSDataType.INT64, getLong(index))); + getTime(index), Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(getLong(index))); } @Override protected TimeValuePair getTimeValuePair( int index, long time, Integer floatPrecision, TSEncoding encoding) { - return new TimeValuePair(time, TsPrimitiveType.getByType(TSDataType.INT64, getLong(index))); + return new TimeValuePair( + time, Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(getLong(index))); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java index 4d1adece4b80..f4f0d06c1bd0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java @@ -19,17 +19,17 @@ package org.apache.iotdb.db.utils.datastructure; -import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; import org.apache.tsfile.write.chunk.IChunkWriter; import org.apache.tsfile.write.chunk.ValueChunkWriter; @@ -47,6 +47,7 @@ public class MergeSortMultiAlignedTVListIterator extends MultiAlignedTVListItera private final Set probeIterators; private final int[] iteratorIndices; private final int[] rowIndices; + private final List valueWriters; private final BitMap bitMap; // Min-Heap: minimal timestamp; if same timestamp, maximum TVList index @@ -85,6 +86,13 @@ public MergeSortMultiAlignedTVListIterator( this.bitMap = new BitMap(tsDataTypeList.size()); this.iteratorIndices = new int[tsDataTypeList.size()]; this.rowIndices = new int[tsDataTypeList.size()]; + this.valueWriters = + tsDataTypeList.stream() + .map( + dataType -> + TypeServices.StorageEngine.ALIGNED_TV_LIST_CHUNK_WRITER_SERVICE.call( + Type.fromTsDataType(dataType))) + .collect(Collectors.toList()); this.ignoreAllNullRows = ignoreAllNullRows; this.heap = new PriorityQueue<>( @@ -243,52 +251,10 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo continue; } - switch (tsDataTypeList.get(columnIndex)) { - case BOOLEAN: - valueChunkWriter.write( - currentTime, - alignedTVList.getBooleanByValueIndex(valueIndex, validColumnIndex), - false); - break; - case INT32: - case DATE: - valueChunkWriter.write( - currentTime, alignedTVList.getIntByValueIndex(valueIndex, validColumnIndex), false); - break; - case INT64: - case TIMESTAMP: - valueChunkWriter.write( - currentTime, - alignedTVList.getLongByValueIndex(valueIndex, validColumnIndex), - false); - break; - case FLOAT: - valueChunkWriter.write( - currentTime, - alignedTVList.getFloatByValueIndex(valueIndex, validColumnIndex), - false); - break; - case DOUBLE: - valueChunkWriter.write( - currentTime, - alignedTVList.getDoubleByValueIndex(valueIndex, validColumnIndex), - false); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - valueChunkWriter.write( - currentTime, - alignedTVList.getBinaryByValueIndex(valueIndex, validColumnIndex), - false); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - tsDataTypeList.get(columnIndex))); - } + valueWriters + .get(columnIndex) + .write( + valueChunkWriter, currentTime, alignedTVList, valueIndex, validColumnIndex, false); } next(); encodeInfo.pointNumInPage++; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java index cf9862ac210f..532d9cc1c017 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java @@ -19,17 +19,16 @@ package org.apache.iotdb.db.utils.datastructure; -import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.IChunkWriter; @@ -38,8 +37,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.apache.iotdb.db.utils.MemUtils.getBinarySize; - public class MergeSortMultiTVListIterator extends MultiTVListIterator { private final List probeIterators; @@ -132,6 +129,9 @@ protected void next() { @Override public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, long[] times) { ChunkWriterImpl chunkWriterImpl = (ChunkWriterImpl) chunkWriter; + TypeServices.TVListChunkWriter valueWriter = + TypeServices.StorageEngine.TV_LIST_CHUNK_WRITER_SERVICE.call( + Type.fromTsDataType(tsDataType)); while (hasNextTimeValuePair()) { // remember current iterator and row index TVList.TVListIterator currIterator = tvListIterators.get(iteratorIndex); @@ -144,43 +144,8 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo chunkWriterImpl.setLastPoint(true); } - switch (tsDataType) { - case BOOLEAN: - chunkWriterImpl.write(time, currIterator.getTVList().getBoolean(row)); - encodeInfo.dataSizeInChunk += 8L + 1L; - break; - case INT32: - case DATE: - chunkWriterImpl.write(time, currIterator.getTVList().getInt(row)); - encodeInfo.dataSizeInChunk += 8L + 4L; - break; - case INT64: - case TIMESTAMP: - chunkWriterImpl.write(time, currIterator.getTVList().getLong(row)); - encodeInfo.dataSizeInChunk += 8L + 8L; - break; - case FLOAT: - chunkWriterImpl.write(time, currIterator.getTVList().getFloat(row)); - encodeInfo.dataSizeInChunk += 8L + 4L; - break; - case DOUBLE: - chunkWriterImpl.write(time, currIterator.getTVList().getDouble(row)); - encodeInfo.dataSizeInChunk += 8L + 8L; - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - Binary value = currIterator.getTVList().getBinary(row); - chunkWriterImpl.write(time, value); - encodeInfo.dataSizeInChunk += 8L + getBinarySize(value); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - tsDataType)); - } + encodeInfo.dataSizeInChunk += + valueWriter.write(chunkWriterImpl, time, currIterator.getTVList(), row); encodeInfo.pointNumInChunk++; if (encodeInfo.pointNumInChunk >= encodeInfo.maxNumberOfPointsInChunk diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java index 83bfa8c6888b..68fc006d93b5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java @@ -19,9 +19,9 @@ package org.apache.iotdb.db.utils.datastructure; -import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; @@ -32,16 +32,18 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.TsBlockUtil; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.series.PaginationController; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.function.LongConsumer; +import static org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE; + public abstract class MultiAlignedTVListIterator extends MemPointIterator { protected List tsDataTypeList; protected List columnIndexList; @@ -161,7 +163,8 @@ public TimeValuePair nextTimeValuePair() { iterator.getPrimitiveTypeObject(currentRowIndex(columnIndex), columnIndex); } TimeValuePair currentTvPair = - new TimeValuePair(currentTime, TsPrimitiveType.getByType(TSDataType.VECTOR, vector)); + new TimeValuePair( + currentTime, Type.fromTsDataType(TSDataType.VECTOR).getTsPrimitiveType(vector)); next(); return currentTvPair; } @@ -181,7 +184,8 @@ public TimeValuePair currentTimeValuePair() { vector[columnIndex] = iterator.getPrimitiveTypeObject(currentRowIndex(columnIndex), columnIndex); } - return new TimeValuePair(currentTime, TsPrimitiveType.getByType(TSDataType.VECTOR, vector)); + return new TimeValuePair( + currentTime, Type.fromTsDataType(TSDataType.VECTOR).getTsPrimitiveType(vector)); } @Override @@ -226,49 +230,22 @@ public TsBlock nextBatch() { continue; } - switch (tsDataTypeList.get(columnIndex)) { - case BOOLEAN: - valueBuilder.writeBoolean( - alignedTVList.getBooleanByValueIndex(valueIndex, validColumnIndex)); - break; - case INT32: - case DATE: - valueBuilder.writeInt(alignedTVList.getIntByValueIndex(valueIndex, validColumnIndex)); - break; - case INT64: - case TIMESTAMP: - valueBuilder.writeLong(alignedTVList.getLongByValueIndex(valueIndex, validColumnIndex)); - break; - case FLOAT: - float valueF = alignedTVList.getFloatByValueIndex(valueIndex, validColumnIndex); - if (encodingList != null) { - valueF = - alignedTVList.roundValueWithGivenPrecision( - valueF, floatPrecision, encodingList.get(columnIndex)); - } - valueBuilder.writeFloat(valueF); - break; - case DOUBLE: - double valueD = alignedTVList.getDoubleByValueIndex(valueIndex, validColumnIndex); - if (encodingList != null) { - valueD = - alignedTVList.roundValueWithGivenPrecision( - valueD, floatPrecision, encodingList.get(columnIndex)); - } - valueBuilder.writeDouble(valueD); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueBuilder.writeBinary( - alignedTVList.getBinaryByValueIndex(valueIndex, validColumnIndex)); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - tsDataTypeList.get(columnIndex))); + TSDataType sourceDataType = alignedTVList.dataTypes.get(validColumnIndex); + TSDataType targetDataType = tsDataTypeList.get(columnIndex); + if (sourceDataType == targetDataType) { + TypeServices.StorageEngine.ARRAY_VALUE_COLUMN_WRITER_SERVICE + .call(Type.fromTsDataType(targetDataType)) + .write( + valueBuilder, + alignedTVList.values.get(validColumnIndex).get(valueIndex / ARRAY_SIZE), + valueIndex % ARRAY_SIZE, + floatPrecision, + encodingList == null ? null : encodingList.get(columnIndex)); + } else { + TsPrimitiveType value = + alignedTVListIterator.getPrimitiveTypeObject( + currentRowIndex(columnIndex), columnIndex); + Type.fromTsDataType(targetDataType).write(valueBuilder, value); } } next(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java index 8895235ffd8f..576a7979f8f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java @@ -19,9 +19,9 @@ package org.apache.iotdb.db.utils.datastructure; -import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -29,10 +29,9 @@ import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.series.PaginationController; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.util.ArrayList; import java.util.Collections; @@ -43,6 +42,7 @@ public abstract class MultiTVListIterator extends MemPointIterator { protected List tvListIterators; protected int floatPrecision; protected TSEncoding encoding; + protected final TypeServices.TVListBatchWriter batchWriter; protected boolean probeNext = false; protected boolean hasNext = false; @@ -102,6 +102,9 @@ protected MultiTVListIterator( } this.floatPrecision = floatPrecision != null ? floatPrecision : 0; this.encoding = encoding; + this.batchWriter = + TypeServices.StorageEngine.TV_LIST_BATCH_WRITER_SERVICE.call( + Type.fromTsDataType(tsDataType)); this.maxNumberOfPointsInPage = maxNumberOfPointsInPage; } @@ -149,111 +152,21 @@ public boolean hasNextBatch() { public TsBlock nextBatch() { TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(tsDataType)); long filteredRowsByPushDownFilter = 0; - switch (tsDataType) { - case BOOLEAN: - while (hasNextTimeValuePair() && builder.getPositionCount() < maxNumberOfPointsInPage) { - TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex); - boolean aBoolean = iterator.getTVList().getBoolean(iterator.getScanOrderIndex(rowIndex)); - if (pushDownFilter == null || pushDownFilter.satisfyBoolean(currentTime, aBoolean)) { - builder.getTimeColumnBuilder().writeLong(currentTime); - builder.getColumnBuilder(0).writeBoolean(aBoolean); - builder.declarePosition(); - } else { - filteredRowsByPushDownFilter++; - } - next(); - } - break; - case INT32: - case DATE: - while (hasNextTimeValuePair() && builder.getPositionCount() < maxNumberOfPointsInPage) { - TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex); - int anInt = iterator.getTVList().getInt(iterator.getScanOrderIndex(rowIndex)); - if (pushDownFilter == null || pushDownFilter.satisfyInteger(currentTime, anInt)) { - builder.getTimeColumnBuilder().writeLong(currentTime); - builder.getColumnBuilder(0).writeInt(anInt); - builder.declarePosition(); - } else { - filteredRowsByPushDownFilter++; - } - next(); - } - break; - case INT64: - case TIMESTAMP: - while (hasNextTimeValuePair() && builder.getPositionCount() < maxNumberOfPointsInPage) { - TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex); - long aLong = iterator.getTVList().getLong(iterator.getScanOrderIndex(rowIndex)); - if (pushDownFilter == null || pushDownFilter.satisfyLong(currentTime, aLong)) { - builder.getTimeColumnBuilder().writeLong(currentTime); - builder.getColumnBuilder(0).writeLong(aLong); - builder.declarePosition(); - } else { - filteredRowsByPushDownFilter++; - } - next(); - } - break; - case FLOAT: - while (hasNextTimeValuePair() && builder.getPositionCount() < maxNumberOfPointsInPage) { - TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex); - TVList floatTvList = iterator.getTVList(); - float aFloat = - floatTvList.roundValueWithGivenPrecision( - floatTvList.getFloat(iterator.getScanOrderIndex(rowIndex)), - floatPrecision, - encoding); - if (pushDownFilter == null || pushDownFilter.satisfyFloat(currentTime, aFloat)) { - builder.getTimeColumnBuilder().writeLong(currentTime); - builder.getColumnBuilder(0).writeFloat(aFloat); - builder.declarePosition(); - } else { - filteredRowsByPushDownFilter++; - } - next(); - } - break; - case DOUBLE: - while (hasNextTimeValuePair() && builder.getPositionCount() < maxNumberOfPointsInPage) { - TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex); - TVList doubleTvList = iterator.getTVList(); - double aDouble = - doubleTvList.roundValueWithGivenPrecision( - doubleTvList.getDouble(iterator.getScanOrderIndex(rowIndex)), - floatPrecision, - encoding); - if (pushDownFilter == null || pushDownFilter.satisfyDouble(currentTime, aDouble)) { - builder.getTimeColumnBuilder().writeLong(currentTime); - builder.getColumnBuilder(0).writeDouble(aDouble); - builder.declarePosition(); - } else { - filteredRowsByPushDownFilter++; - } - next(); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - while (hasNextTimeValuePair() && builder.getPositionCount() < maxNumberOfPointsInPage) { - TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex); - Binary binary = iterator.getTVList().getBinary(iterator.getScanOrderIndex(rowIndex)); - if (pushDownFilter == null || pushDownFilter.satisfyBinary(currentTime, binary)) { - builder.getTimeColumnBuilder().writeLong(currentTime); - builder.getColumnBuilder(0).writeBinary(binary); - builder.declarePosition(); - } else { - filteredRowsByPushDownFilter++; - } - next(); - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - tsDataType)); + while (hasNextTimeValuePair() && builder.getPositionCount() < maxNumberOfPointsInPage) { + TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex); + TVList tvList = iterator.getTVList(); + if (!batchWriter.write( + tvList, + iterator.getScanOrderIndex(rowIndex), + currentTime, + pushDownFilter, + builder, + floatPrecision, + encoding, + null)) { + filteredRowsByPushDownFilter++; + } + next(); } if (this.getQueryContext().isVerbose() && filteredRowsByPushDownFilter > 0) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index 506f82d9e10e..8baa504d2419 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager; import org.apache.iotdb.db.utils.MathUtils; +import org.apache.iotdb.db.utils.TypeServices; import com.google.common.collect.ImmutableList; import org.apache.tsfile.enums.TSDataType; @@ -36,11 +37,11 @@ import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.IChunkWriter; @@ -56,7 +57,6 @@ import java.util.concurrent.locks.ReentrantLock; import static org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE; -import static org.apache.iotdb.db.utils.MemUtils.getBinarySize; import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted; import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF; @@ -166,29 +166,9 @@ protected TVList() { } public static TVList newList(TSDataType dataType) { - switch (dataType) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return BinaryTVList.newList(); - case FLOAT: - return FloatTVList.newList(); - case INT32: - return IntTVList.newList(TSDataType.INT32); - case DATE: - return IntTVList.newList(TSDataType.DATE); - case INT64: - case TIMESTAMP: - return LongTVList.newList(); - case DOUBLE: - return DoubleTVList.newList(); - case BOOLEAN: - return BooleanTVList.newList(); - default: - break; - } - return null; + return TypeServices.StorageEngine.TV_LIST_PROVIDER_SERVICE + .call(Type.fromTsDataType(dataType)) + .newList(); } // get array memory cost of working TVList @@ -753,56 +733,16 @@ protected double roundValueWithGivenPrecision( public static TVList deserialize(DataInputStream stream) throws IOException { TSDataType dataType = ReadWriteIOUtils.readDataType(stream); - switch (dataType) { - case TEXT: - case BLOB: - case OBJECT: - case STRING: - return BinaryTVList.deserialize(stream); - case FLOAT: - return FloatTVList.deserialize(stream); - case INT32: - return IntTVList.deserialize(stream, TSDataType.INT32); - case DATE: - return IntTVList.deserialize(stream, TSDataType.DATE); - case INT64: - case TIMESTAMP: - return LongTVList.deserialize(stream); - case DOUBLE: - return DoubleTVList.deserialize(stream); - case BOOLEAN: - return BooleanTVList.deserialize(stream); - default: - break; - } - return null; + return TypeServices.StorageEngine.TV_LIST_PROVIDER_SERVICE + .call(Type.fromTsDataType(dataType)) + .deserialize(stream); } public static TVList deserializeWithoutBitMap(DataInputStream stream) throws IOException { TSDataType dataType = ReadWriteIOUtils.readDataType(stream); - switch (dataType) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return BinaryTVList.deserializeWithoutBitMap(stream); - case FLOAT: - return FloatTVList.deserializeWithoutBitMap(stream); - case INT32: - return IntTVList.deserializeWithoutBitMap(stream, TSDataType.INT32); - case DATE: - return IntTVList.deserializeWithoutBitMap(stream, TSDataType.DATE); - case INT64: - case TIMESTAMP: - return LongTVList.deserializeWithoutBitMap(stream); - case DOUBLE: - return DoubleTVList.deserializeWithoutBitMap(stream); - case BOOLEAN: - return BooleanTVList.deserializeWithoutBitMap(stream); - default: - break; - } - return null; + return TypeServices.StorageEngine.TV_LIST_PROVIDER_SERVICE + .call(Type.fromTsDataType(dataType)) + .deserializeWithoutBitMap(stream); } public List getTimestamps() { @@ -878,6 +818,7 @@ public class TVListIterator extends MemPointIterator { private final int[] deleteCursor; private final int floatPrecision; private final TSEncoding encoding; + private final TypeServices.TVListBatchWriter batchWriter; // used by nextBatch during query protected final int maxNumberOfPointsInPage; @@ -896,6 +837,12 @@ public TVListIterator( this.deletionList = deletionList; this.floatPrecision = floatPrecision != null ? floatPrecision : 0; this.encoding = encoding; + TSDataType dataType = getDataType(); + this.batchWriter = + dataType == TSDataType.VECTOR + ? null + : TypeServices.StorageEngine.TV_LIST_BATCH_WRITER_SERVICE.call( + Type.fromTsDataType(dataType)); this.index = 0; this.rows = rowCount; this.probeNext = false; @@ -1109,189 +1056,28 @@ public TsBlock nextBatch() { : null; long filteredRowsByPushDownFilter = 0; - switch (dataType) { - case BOOLEAN: - while (index < rows - && builder.getPositionCount() < maxNumberOfPointsInPage - && paginationController.hasCurLimit()) { - long time = getTime(getScanOrderIndex(index)); - if (isCurrentTimeExceedTimeRange(time)) { - break; - } - if (!isInvalidRow( - time, index, deletionList, deleteCursor, scanOrder, filteredRowsByTimeFilter)) { - boolean aBoolean = getBoolean(getScanOrderIndex(index)); - if (pushDownFilter == null || pushDownFilter.satisfyBoolean(time, aBoolean)) { - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - index++; - continue; - } - paginationController.consumeLimit(); - builder.getTimeColumnBuilder().writeLong(time); - builder.getColumnBuilder(0).writeBoolean(aBoolean); - builder.declarePosition(); - } else { - filteredRowsByPushDownFilter++; - } - } - index++; - } - break; - case INT32: - case DATE: - while (index < rows - && builder.getPositionCount() < maxNumberOfPointsInPage - && paginationController.hasCurLimit()) { - long time = getTime(getScanOrderIndex(index)); - if (isCurrentTimeExceedTimeRange(time)) { - break; - } - if (!isInvalidRow( - time, index, deletionList, deleteCursor, scanOrder, filteredRowsByTimeFilter)) { - int anInt = getInt(getScanOrderIndex(index)); - if (pushDownFilter == null || pushDownFilter.satisfyInteger(time, anInt)) { - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - index++; - continue; - } - paginationController.consumeLimit(); - builder.getTimeColumnBuilder().writeLong(time); - builder.getColumnBuilder(0).writeInt(anInt); - builder.declarePosition(); - } else { - filteredRowsByPushDownFilter++; - } - } - index++; - } - break; - case INT64: - case TIMESTAMP: - while (index < rows - && builder.getPositionCount() < maxNumberOfPointsInPage - && paginationController.hasCurLimit()) { - long time = getTime(getScanOrderIndex(index)); - if (isCurrentTimeExceedTimeRange(time)) { - break; - } - if (!isInvalidRow( - time, index, deletionList, deleteCursor, scanOrder, filteredRowsByTimeFilter)) { - long aLong = getLong(getScanOrderIndex(index)); - if (pushDownFilter == null || pushDownFilter.satisfyLong(time, aLong)) { - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - index++; - continue; - } - paginationController.consumeLimit(); - builder.getTimeColumnBuilder().writeLong(time); - builder.getColumnBuilder(0).writeLong(aLong); - builder.declarePosition(); - } else { - filteredRowsByPushDownFilter++; - } - } - index++; - } + while (index < rows + && builder.getPositionCount() < maxNumberOfPointsInPage + && paginationController.hasCurLimit()) { + int scanOrderIndex = getScanOrderIndex(index); + long time = getTime(scanOrderIndex); + if (isCurrentTimeExceedTimeRange(time)) { break; - case FLOAT: - while (index < rows - && builder.getPositionCount() < maxNumberOfPointsInPage - && paginationController.hasCurLimit()) { - long time = getTime(getScanOrderIndex(index)); - if (isCurrentTimeExceedTimeRange(time)) { - break; - } - if (!isInvalidRow( - time, index, deletionList, deleteCursor, scanOrder, filteredRowsByTimeFilter)) { - float aFloat = - roundValueWithGivenPrecision( - getFloat(getScanOrderIndex(index)), floatPrecision, encoding); - if (pushDownFilter == null || pushDownFilter.satisfyFloat(time, aFloat)) { - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - index++; - continue; - } - paginationController.consumeLimit(); - builder.getTimeColumnBuilder().writeLong(time); - builder.getColumnBuilder(0).writeFloat(aFloat); - builder.declarePosition(); - } else { - filteredRowsByPushDownFilter++; - } - } - index++; - } - break; - case DOUBLE: - while (index < rows - && builder.getPositionCount() < maxNumberOfPointsInPage - && paginationController.hasCurLimit()) { - long time = getTime(getScanOrderIndex(index)); - if (isCurrentTimeExceedTimeRange(time)) { - break; - } - if (!isInvalidRow( - time, index, deletionList, deleteCursor, scanOrder, filteredRowsByTimeFilter)) { - double aDouble = - roundValueWithGivenPrecision( - getDouble(getScanOrderIndex(index)), floatPrecision, encoding); - if (pushDownFilter == null || pushDownFilter.satisfyDouble(time, aDouble)) { - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - index++; - continue; - } - paginationController.consumeLimit(); - builder.getTimeColumnBuilder().writeLong(time); - builder.getColumnBuilder(0).writeDouble(aDouble); - builder.declarePosition(); - } else { - filteredRowsByPushDownFilter++; - } - } - index++; - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - while (index < rows - && builder.getPositionCount() < maxNumberOfPointsInPage - && paginationController.hasCurLimit()) { - long time = getTime(getScanOrderIndex(index)); - if (isCurrentTimeExceedTimeRange(time)) { - break; - } - if (!isInvalidRow( - time, index, deletionList, deleteCursor, scanOrder, filteredRowsByTimeFilter)) { - Binary binary = getBinary(getScanOrderIndex(index)); - if (pushDownFilter == null || pushDownFilter.satisfyBinary(time, binary)) { - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - index++; - continue; - } - paginationController.consumeLimit(); - builder.getTimeColumnBuilder().writeLong(time); - builder.getColumnBuilder(0).writeBinary(binary); - builder.declarePosition(); - } else { - filteredRowsByPushDownFilter++; - } - } - index++; - } - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataType)); + } + if (!isInvalidRow( + time, index, deletionList, deleteCursor, scanOrder, filteredRowsByTimeFilter) + && !batchWriter.write( + outer, + scanOrderIndex, + time, + pushDownFilter, + builder, + floatPrecision, + encoding, + paginationController)) { + filteredRowsByPushDownFilter++; + } + index++; } // count the filtered row from time filter and other filter @@ -1352,8 +1138,10 @@ public int getScanOrderIndex(int rowIndex) { @Override public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, long[] times) { - TSDataType dataType = getDataType(); ChunkWriterImpl chunkWriterImpl = (ChunkWriterImpl) chunkWriter; + TypeServices.TVListChunkWriter valueWriter = + TypeServices.StorageEngine.TV_LIST_CHUNK_WRITER_SERVICE.call( + Type.fromTsDataType(getDataType())); for (; index < rows; index++) { if (isNullValue(getValueIndex(index))) { continue; @@ -1373,43 +1161,7 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo } } - switch (dataType) { - case BOOLEAN: - chunkWriterImpl.write(time, getBoolean(index)); - encodeInfo.dataSizeInChunk += 8L + 1L; - break; - case INT32: - case DATE: - chunkWriterImpl.write(time, getInt(index)); - encodeInfo.dataSizeInChunk += 8L + 4L; - break; - case INT64: - case TIMESTAMP: - chunkWriterImpl.write(time, getLong(index)); - encodeInfo.dataSizeInChunk += 8L + 8L; - break; - case FLOAT: - chunkWriterImpl.write(time, getFloat(index)); - encodeInfo.dataSizeInChunk += 8L + 4L; - break; - case DOUBLE: - chunkWriterImpl.write(time, getDouble(index)); - encodeInfo.dataSizeInChunk += 8L + 8L; - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary value = getBinary(index); - chunkWriterImpl.write(time, value); - encodeInfo.dataSizeInChunk += 8L + getBinarySize(value); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - DataNodeMiscMessages.MISC_EXCEPTION_DATA_TYPE_S_IS_NOT_SUPPORTED_5D5C02E4, - dataType)); - } + encodeInfo.dataSizeInChunk += valueWriter.write(chunkWriterImpl, time, TVList.this, index); encodeInfo.pointNumInChunk++; if (encodeInfo.pointNumInChunk >= encodeInfo.maxNumberOfPointsInChunk || encodeInfo.dataSizeInChunk >= encodeInfo.targetChunkSize) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java index a87fb480d9d2..22550fe292cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java @@ -19,11 +19,12 @@ package org.apache.iotdb.db.utils.windowing.window; +import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.db.utils.windowing.api.Window; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.write.UnSupportedDataTypeException; public class WindowImpl implements Window { @@ -51,51 +52,9 @@ private void init(EvictableBatchList list, int begin) { timestamps[i] = list.getTimeByIndex(begin + i); } - switch (dataType) { - case INT32: - case DATE: - intValues = new int[size]; - for (int i = 0; i < size; ++i) { - intValues[i] = list.getIntByIndex(begin + i); - } - break; - case INT64: - case TIMESTAMP: - longValues = new long[size]; - for (int i = 0; i < size; ++i) { - longValues[i] = list.getLongByIndex(begin + i); - } - break; - case FLOAT: - floatValues = new float[size]; - for (int i = 0; i < size; ++i) { - floatValues[i] = list.getFloatByIndex(begin + i); - } - break; - case DOUBLE: - doubleValues = new double[size]; - for (int i = 0; i < size; ++i) { - doubleValues[i] = list.getDoubleByIndex(begin + i); - } - break; - case BOOLEAN: - booleanValues = new boolean[size]; - for (int i = 0; i < size; ++i) { - booleanValues[i] = list.getBooleanByIndex(begin + i); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - binaryValues = new Binary[size]; - for (int i = 0; i < size; ++i) { - binaryValues[i] = list.getBinaryByIndex(begin + i); - } - break; - default: - throw new UnSupportedDataTypeException(dataType.toString()); - } + TypeServices.StorageEngine.WINDOW_VALUE_ARRAY_BUILDER_SERVICE + .call(Type.fromTsDataType(dataType)) + .build(this, list, begin, size); } @Override @@ -207,4 +166,28 @@ public void setBoolean(int index, boolean value) { public void setBinary(int index, Binary value) { binaryValues[index] = value; } + + public void setIntValues(int[] intValues) { + this.intValues = intValues; + } + + public void setLongValues(long[] longValues) { + this.longValues = longValues; + } + + public void setFloatValues(float[] floatValues) { + this.floatValues = floatValues; + } + + public void setDoubleValues(double[] doubleValues) { + this.doubleValues = doubleValues; + } + + public void setBooleanValues(boolean[] booleanValues) { + this.booleanValues = booleanValues; + } + + public void setBinaryValues(Binary[] binaryValues) { + this.binaryValues = binaryValues; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowTest.java new file mode 100644 index 000000000000..3699693f1ed5 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.pipe.event.common.row; + +import org.apache.iotdb.pipe.api.type.Binary; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BytesUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.time.LocalDate; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class PipeRowTest { + + @Test + public void testGetObjectForSupportedTypes() { + final LocalDate date = LocalDate.of(2026, 8, 12); + final PipeRow row = + new PipeRow( + 0, + "root.test.device", + false, + null, + new long[] {1L}, + new TSDataType[] { + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.DATE, + TSDataType.INT64, + TSDataType.TIMESTAMP, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.BLOB, + TSDataType.STRING + }, + new Object[] { + new boolean[] {true}, + new int[] {2}, + new LocalDate[] {date}, + new long[] {3L}, + new long[] {4L}, + new float[] {5.5F}, + new double[] {6.5D}, + new org.apache.tsfile.utils.Binary[] {BytesUtils.valueOf("text")}, + new org.apache.tsfile.utils.Binary[] {BytesUtils.valueOf("blob")}, + new org.apache.tsfile.utils.Binary[] {BytesUtils.valueOf("string")} + }, + null, + new String[] { + "boolean", + "int32", + "date", + "int64", + "timestamp", + "float", + "double", + "text", + "blob", + "string" + }); + + assertEquals(Boolean.TRUE, row.getObject(0)); + assertEquals(2, row.getObject(1)); + assertEquals(date, row.getObject(2)); + assertEquals(3L, row.getObject(3)); + assertEquals(4L, row.getObject(4)); + assertEquals(5.5F, row.getObject(5)); + assertEquals(6.5D, row.getObject(6)); + assertBinaryEquals("text", row.getObject(7)); + assertBinaryEquals("blob", row.getObject(8)); + assertBinaryEquals("string", row.getObject(9)); + } + + @Test + public void testGetObjectForUnsupportedType() { + final PipeRow row = + new PipeRow( + 0, + "root.test.device", + false, + null, + new long[] {1L}, + new TSDataType[] {TSDataType.OBJECT}, + new Object[] {new Object[] {new Object()}}, + null, + new String[] {"object"}); + + Assert.assertThrows(IllegalArgumentException.class, () -> row.getObject(0)); + } + + private static void assertBinaryEquals(final String expected, final Object actual) { + assertArrayEquals(Binary.stringToBytes(expected), ((Binary) actual).getValues()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/client/IoTDBOpcUaClientTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/client/IoTDBOpcUaClientTest.java index d327155f32d0..e319319a025a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/client/IoTDBOpcUaClientTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/client/IoTDBOpcUaClientTest.java @@ -25,8 +25,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -286,7 +286,8 @@ private static Pair lastValue( final String measurement, final long timestamp, final long value) { return new Pair<>( new MeasurementSchema(measurement, TSDataType.INT64), - new TimeValuePair(timestamp, TsPrimitiveType.getByType(TSDataType.INT64, value))); + new TimeValuePair( + timestamp, Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(value))); } private static ArgumentMatcher> nodeIds(final String... identifiers) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaNameSpaceMetadataTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaNameSpaceMetadataTest.java index a7410aea71ac..a59b5ae3eb17 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaNameSpaceMetadataTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaNameSpaceMetadataTest.java @@ -25,10 +25,10 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.eclipse.milo.opcua.stack.core.types.builtin.DateTime; @@ -137,7 +137,8 @@ private static TimeValuePair timeValue( ? new Binary( String.valueOf(value), org.apache.tsfile.common.conf.TSFileConfig.STRING_CHARSET) : value; - return new TimeValuePair(timestamp, TsPrimitiveType.getByType(dataType, primitiveValue)); + return new TimeValuePair( + timestamp, Type.fromTsDataType(dataType).getTsPrimitiveType(primitiveValue)); } private static class CapturedRow { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToFilterVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToFilterVisitorTest.java new file mode 100644 index 000000000000..f4ad2044bc2a --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToFilterVisitorTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.expression.visitor.predicate; + +import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.junit.Assert; +import org.junit.Test; + +public class ConvertPredicateToFilterVisitorTest { + + @Test + public void testGetValue() { + Assert.assertEquals( + Integer.valueOf(1), + ConvertPredicateToFilterVisitor.getValue("1", TSDataType.INT32)); + Assert.assertEquals( + Long.valueOf(2), ConvertPredicateToFilterVisitor.getValue("2", TSDataType.INT64)); + Assert.assertEquals( + Long.valueOf(3), ConvertPredicateToFilterVisitor.getValue("3", TSDataType.TIMESTAMP)); + Assert.assertEquals( + Float.valueOf(4.5F), + ConvertPredicateToFilterVisitor.getValue("4.5", TSDataType.FLOAT)); + Assert.assertEquals( + Double.valueOf(6.75), + ConvertPredicateToFilterVisitor.getValue("6.75", TSDataType.DOUBLE)); + Assert.assertEquals( + Boolean.TRUE, + ConvertPredicateToFilterVisitor.getValue("TrUe", TSDataType.BOOLEAN)); + Assert.assertEquals( + Boolean.FALSE, + ConvertPredicateToFilterVisitor.getValue("FaLsE", TSDataType.BOOLEAN)); + Assert.assertEquals( + new Binary(new byte[] {(byte) 0xCA, (byte) 0xFE}), + ConvertPredicateToFilterVisitor.getValue("CAFE", TSDataType.BLOB)); + Assert.assertEquals( + new Binary("text", TSFileConfig.STRING_CHARSET), + ConvertPredicateToFilterVisitor.getValue("text", TSDataType.TEXT)); + Assert.assertEquals( + new Binary("string", TSFileConfig.STRING_CHARSET), + ConvertPredicateToFilterVisitor.getValue("string", TSDataType.STRING)); + Assert.assertEquals( + Integer.valueOf(DateTimeUtils.parseDateExpressionToInt("2026-07-16")), + ConvertPredicateToFilterVisitor.getValue("2026-07-16", TSDataType.DATE)); + } + + @Test + public void testGetValueWithInvalidValue() { + IllegalArgumentException numberException = + Assert.assertThrows( + IllegalArgumentException.class, + () -> ConvertPredicateToFilterVisitor.getValue("invalid", TSDataType.INT32)); + Assert.assertEquals( + String.format( + DataNodeQueryMessages.EXCEPTION_ARG_CANNOT_BE_CAST_TO_ARG_28F0C5FC, + "invalid", + TSDataType.INT32), + numberException.getMessage()); + + IllegalArgumentException booleanException = + Assert.assertThrows( + IllegalArgumentException.class, + () -> ConvertPredicateToFilterVisitor.getValue("invalid", TSDataType.BOOLEAN)); + Assert.assertEquals( + String.format( + DataNodeQueryMessages.EXCEPTION_ARG_CANNOT_BE_CAST_TO_ARG_28F0C5FC, + "invalid", + TSDataType.BOOLEAN), + booleanException.getMessage()); + } + + @Test + public void testGetValueWithUnsupportedType() { + UnsupportedOperationException exception = + Assert.assertThrows( + UnsupportedOperationException.class, + () -> ConvertPredicateToFilterVisitor.getValue("value", TSDataType.VECTOR)); + Assert.assertEquals( + String.format( + DataNodeQueryMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_751BF348, TSDataType.VECTOR), + exception.getMessage()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/RowGetDoubleTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/RowGetDoubleTest.java new file mode 100644 index 000000000000..26d88281257a --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/RowGetDoubleTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.adapter; + +import org.apache.iotdb.udf.api.access.Row; +import org.apache.iotdb.udf.api.utils.RowImpl; + +import org.apache.tsfile.enums.TSDataType; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class RowGetDoubleTest { + + @Test + public void testNumericConversion() throws IOException { + // Both Object[] implementations must widen all four numeric types, including large longs. + for (Row row : + createRows( + new TSDataType[] { + TSDataType.INT32, TSDataType.INT64, TSDataType.FLOAT, TSDataType.DOUBLE + }, + new Object[] {Integer.MIN_VALUE, Long.MAX_VALUE, 0.1f, -1.25d, 123L})) { + assertEquals((double) Integer.MIN_VALUE, row.getDouble(0), 0d); + assertEquals((double) Long.MAX_VALUE, row.getDouble(1), 0d); + assertEquals((double) 0.1f, row.getDouble(2), 0d); + assertEquals(-1.25d, row.getDouble(3), 0d); + assertEquals(123L, row.getTime()); + assertEquals(4, row.size()); + } + } + + @Test + public void testSpecialFloatingPointValues() throws IOException { + // Widening must preserve NaN, infinities, and the sign of zero. + for (Row row : + createRows( + new TSDataType[] { + TSDataType.FLOAT, TSDataType.FLOAT, TSDataType.DOUBLE, TSDataType.DOUBLE + }, + new Object[] { + Float.NaN, Float.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, -0.0d, 123L + })) { + assertTrue(Double.isNaN(row.getDouble(0))); + assertEquals(Double.POSITIVE_INFINITY, row.getDouble(1), 0d); + assertEquals(Double.NEGATIVE_INFINITY, row.getDouble(2), 0d); + assertEquals(Double.doubleToLongBits(-0.0d), Double.doubleToLongBits(row.getDouble(3))); + } + } + + @Test + public void testInvalidReads() throws IOException { + // Null, nonnumeric values, and invalid array indices retain their failure behavior. + for (Row row : + createRows( + new TSDataType[] {TSDataType.DOUBLE, TSDataType.BOOLEAN}, + new Object[] {null, true, 123L})) { + assertTrue(row.isNull(0)); + assertThrows(NullPointerException.class, () -> row.getDouble(0)); + assertThrows(ClassCastException.class, () -> row.getDouble(1)); + assertThrows(IndexOutOfBoundsException.class, () -> row.getDouble(-1)); + assertThrows(IndexOutOfBoundsException.class, () -> row.getDouble(3)); + } + } + + private List createRows(TSDataType[] types, Object[] values) { + // RowImpl includes the trailing timestamp in its schema; the adapter does not. + TSDataType[] typesWithTime = Arrays.copyOf(types, types.length + 1); + typesWithTime[types.length] = TSDataType.TIMESTAMP; + RowImpl row = new RowImpl(typesWithTime); + row.setRowRecord(values); + return Arrays.asList( + row, new ElasticSerializableRowRecordListBackedMultiColumnRow(types).setRowRecord(values)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReaderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReaderTest.java new file mode 100644 index 000000000000..ccab0c8c32d8 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReaderTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.input; + +import org.apache.iotdb.calc.exception.QueryProcessException; +import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand; +import org.apache.iotdb.db.queryengine.transformation.dag.util.TransformUtils; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +public class ConstantInputReaderTest { + + @Test + public void testSupportedConstantColumns() throws Exception { + final TSDataType[] dataTypes = { + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.BOOLEAN + }; + final String[] valueStrings = {"1", "2", "3.5", "4.5", "text", "true"}; + final Object[] expectedValues = { + 1, 2L, 3.5F, 4.5D, new Binary("text", TSFileConfig.STRING_CHARSET), true + }; + + for (int i = 0; i < dataTypes.length; i++) { + final ConstantOperand operand = new ConstantOperand(dataTypes[i], valueStrings[i]); + + final Column inputColumn = new ConstantInputReader(operand).current()[0]; + assertEquals( + TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber(), + inputColumn.getPositionCount()); + assertEquals(expectedValues[i], inputColumn.getObject(0)); + + final Column transformedColumn = TransformUtils.transformConstantOperandToColumn(operand); + assertEquals(1, transformedColumn.getPositionCount()); + assertEquals(expectedValues[i], transformedColumn.getObject(0)); + } + } + + @Test + public void testUnsupportedConstantColumnType() { + final ConstantOperand operand = new ConstantOperand(TSDataType.DATE, "2026-07-17"); + + assertThrows(QueryProcessException.class, () -> new ConstantInputReader(operand)); + assertThrows( + UnSupportedDataTypeException.class, + () -> TransformUtils.transformConstantOperandToColumn(operand)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtilsTest.java new file mode 100644 index 000000000000..a95d779a9640 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtilsTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.util; + +import org.apache.iotdb.calc.exception.QueryProcessException; +import org.apache.iotdb.db.queryengine.transformation.datastructure.util.ValueRecorder; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class TransformUtilsTest { + + @Test + public void testSplitWindowForSupportedTypes() { + assertStateWindowSplit(TSDataType.INT32, 1, 2, 3); + assertStateWindowSplit(TSDataType.INT64, 1L, 2L, 3L); + assertStateWindowSplit(TSDataType.FLOAT, 1F, 2F, 3F); + assertStateWindowSplit(TSDataType.DOUBLE, 1D, 2D, 3D); + assertStateWindowSplit(TSDataType.BOOLEAN, false, false, true); + assertStateWindowSplit( + TSDataType.TEXT, + new Binary("a", TSFileConfig.STRING_CHARSET), + new Binary("a", TSFileConfig.STRING_CHARSET), + new Binary("b", TSFileConfig.STRING_CHARSET)); + } + + @Test + public void testSplitWindowForUnsupportedType() { + final Column values = createColumn(TSDataType.DATE, 20260717, 20260718); + + assertThrows( + UnsupportedOperationException.class, + () -> + TransformUtils.splitWindowForStateWindow( + TSDataType.DATE, new ValueRecorder(), 1, values, 1)); + } + + @Test + public void testTransformationTypeUtils() throws Exception { + assertEquals( + 1D, TypeUtils.castValueToDouble(createColumn(TSDataType.INT32, 1), TSDataType.INT32, 0), 0); + assertEquals( + 2D, TypeUtils.castValueToDouble(createColumn(TSDataType.DATE, 2), TSDataType.DATE, 0), 0); + assertEquals( + 3D, + TypeUtils.castValueToDouble(createColumn(TSDataType.INT64, 3L), TSDataType.INT64, 0), + 0); + assertEquals( + 4D, + TypeUtils.castValueToDouble( + createColumn(TSDataType.TIMESTAMP, 4L), TSDataType.TIMESTAMP, 0), + 0); + assertEquals( + 5.5D, + TypeUtils.castValueToDouble(createColumn(TSDataType.FLOAT, 5.5F), TSDataType.FLOAT, 0), + 0); + assertEquals( + 6.5D, + TypeUtils.castValueToDouble(createColumn(TSDataType.DOUBLE, 6.5D), TSDataType.DOUBLE, 0), + 0); + assertEquals( + 1D, + TypeUtils.castValueToDouble(createColumn(TSDataType.BOOLEAN, true), TSDataType.BOOLEAN, 0), + 0); + + final Column textColumn = + createColumn(TSDataType.TEXT, new Binary("text", TSFileConfig.STRING_CHARSET)); + assertThrows( + QueryProcessException.class, + () -> TypeUtils.castValueToDouble(textColumn, TSDataType.TEXT, 0)); + } + + private static void assertStateWindowSplit( + TSDataType dataType, Object first, Object second, Object third) { + final Column values = createColumn(dataType, first, second, third); + final ValueRecorder valueRecorder = new ValueRecorder(); + + assertFalse(TransformUtils.splitWindowForStateWindow(dataType, valueRecorder, 1, values, 1)); + assertTrue(TransformUtils.splitWindowForStateWindow(dataType, valueRecorder, 1, values, 2)); + } + + private static Column createColumn(TSDataType dataType, Object... values) { + final Type type = Type.fromTsDataType(dataType); + final ColumnBuilder builder = TypeUtils.initColumnBuilder(dataType, values.length); + for (final Object value : values) { + type.writeObject(builder, value); + } + return builder.build(); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriterTest.java index de28c6292157..7d5015700662 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriterTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.ChunkMetadataElement; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.flushcontroller.AbstractCompactionFlushController; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.PageException; @@ -33,14 +34,22 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.IChunkWriter; import org.apache.tsfile.write.schema.MeasurementSchema; import org.apache.tsfile.write.schema.Schema; import org.junit.Assert; import org.junit.Test; import java.io.IOException; +import java.lang.reflect.Method; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.List; public class AbstractCompactionWriterTest { @@ -63,6 +72,105 @@ public void testBinarySizeCheckpointTriggersChunkSizeCheckBeforePointCheckpoint( Assert.assertEquals(1, chunkWriter.chunkSizeCheckCount); } + @Test + public void testEstimateWrittenPointTotalSize() throws Exception { + TestCompactionWriter compactionWriter = new TestCompactionWriter(); + TsPrimitiveType binaryValue = + Type.fromTsDataType(TSDataType.TEXT) + .getTsPrimitiveType(new Binary("abc", TSFileConfig.STRING_CHARSET)); + TsPrimitiveType vector = + Type.fromTsDataType(TSDataType.VECTOR) + .getTsPrimitiveType( + new TsPrimitiveType[] { + Type.fromTsDataType(TSDataType.INT32).getTsPrimitiveType(1), + Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(2L), + Type.fromTsDataType(TSDataType.FLOAT).getTsPrimitiveType(3F), + Type.fromTsDataType(TSDataType.DOUBLE).getTsPrimitiveType(4D), + Type.fromTsDataType(TSDataType.BOOLEAN).getTsPrimitiveType(true), + binaryValue, + null + }); + + Method estimateMethod = + AbstractCompactionWriter.class.getDeclaredMethod( + "estimateWrittenPointTotalSize", TsPrimitiveType.class); + estimateMethod.setAccessible(true); + Assert.assertEquals(36L, estimateMethod.invoke(compactionWriter, vector)); + + CountingChunkWriter chunkWriter = new CountingChunkWriter(); + compactionWriter.startMeasurement("s1", chunkWriter, SUB_TASK_ID); + compactionWriter.writePoint(binaryValue, chunkWriter); + Assert.assertEquals(11L, compactionWriter.getWrittenPointTotalSize()); + + TsBlockBuilder builder = + new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.TEXT, TSDataType.BOOLEAN)); + builder.getTimeColumnBuilder().writeLong(1); + builder.getColumnBuilder(0).writeInt(1); + builder.getColumnBuilder(1).writeBinary(new Binary("abc", TSFileConfig.STRING_CHARSET)); + builder.getColumnBuilder(2).writeBoolean(true); + builder.declarePosition(); + builder.getTimeColumnBuilder().writeLong(2); + builder.getColumnBuilder(0).appendNull(); + builder.getColumnBuilder(1).appendNull(); + builder.getColumnBuilder(2).appendNull(); + builder.declarePosition(); + + Assert.assertEquals(29L, compactionWriter.estimate(builder.build())); + } + + @Test + public void testWriteDataPointForAllDataTypes() { + assertWrittenPointTotalSize(TSDataType.BOOLEAN, true, Long.BYTES + Byte.BYTES); + assertWrittenPointTotalSize(TSDataType.INT32, 1, Long.BYTES + Integer.BYTES); + assertWrittenPointTotalSize(TSDataType.DATE, 1, Long.BYTES + Integer.BYTES); + assertWrittenPointTotalSize(TSDataType.INT64, 1L, Long.BYTES + Long.BYTES); + assertWrittenPointTotalSize(TSDataType.TIMESTAMP, 1L, Long.BYTES + Long.BYTES); + assertWrittenPointTotalSize(TSDataType.FLOAT, 1F, Long.BYTES + Float.BYTES); + assertWrittenPointTotalSize(TSDataType.DOUBLE, 1D, Long.BYTES + Double.BYTES); + + Binary binary = new Binary("abc", TSFileConfig.STRING_CHARSET); + assertWrittenPointTotalSize(TSDataType.TEXT, binary, Long.BYTES + binary.getLength()); + assertWrittenPointTotalSize(TSDataType.STRING, binary, Long.BYTES + binary.getLength()); + assertWrittenPointTotalSize(TSDataType.BLOB, binary, Long.BYTES + binary.getLength()); + assertWrittenPointTotalSize(TSDataType.OBJECT, binary, Long.BYTES + binary.getLength()); + } + + @Test + public void testWriteAlignedDataPointWithFixedLengthTypes() { + TestCompactionWriter compactionWriter = new TestCompactionWriter(); + AlignedChunkWriterImpl chunkWriter = + new AlignedChunkWriterImpl( + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.BOOLEAN))); + TsPrimitiveType vector = + Type.fromTsDataType(TSDataType.VECTOR) + .getTsPrimitiveType( + new TsPrimitiveType[] { + Type.fromTsDataType(TSDataType.INT32).getTsPrimitiveType(1), + Type.fromTsDataType(TSDataType.BOOLEAN).getTsPrimitiveType(true) + }); + + compactionWriter.startMeasurement("aligned", chunkWriter, SUB_TASK_ID); + compactionWriter.writePoint(vector, chunkWriter); + + Assert.assertEquals( + Long.BYTES + Integer.BYTES + Byte.BYTES, compactionWriter.getWrittenPointTotalSize()); + } + + private static void assertWrittenPointTotalSize( + TSDataType dataType, Object value, long expectedSize) { + TestCompactionWriter compactionWriter = new TestCompactionWriter(); + CountingChunkWriter chunkWriter = new CountingChunkWriter(dataType); + TsPrimitiveType primitiveType = Type.fromTsDataType(dataType).getTsPrimitiveType(value); + + compactionWriter.startMeasurement("s1", chunkWriter, SUB_TASK_ID); + compactionWriter.writePoint(primitiveType, chunkWriter); + + Assert.assertEquals( + dataType.toString(), expectedSize, compactionWriter.getWrittenPointTotalSize()); + } + private static PageHeader createPageHeader(int compressedSize) { Statistics statistics = Statistics.getStatsByType(TSDataType.TEXT); return new PageHeader(compressedSize, compressedSize, statistics); @@ -73,7 +181,11 @@ private static class CountingChunkWriter extends ChunkWriterImpl { private int chunkSizeCheckCount; private CountingChunkWriter() { - super(new MeasurementSchema("s1", TSDataType.TEXT)); + this(TSDataType.TEXT); + } + + private CountingChunkWriter(TSDataType dataType) { + super(new MeasurementSchema("s1", dataType)); } @Override @@ -86,6 +198,18 @@ public boolean checkIsChunkSizeOverThreshold( private static class TestCompactionWriter extends AbstractCompactionWriter { + private void writePoint(TsPrimitiveType value, IChunkWriter chunkWriter) { + writeDataPoint(1, value, chunkWriter, SUB_TASK_ID); + } + + private long getWrittenPointTotalSize() { + return writtenPointTotalSizeArray[SUB_TASK_ID]; + } + + private long estimate(TsBlock tsBlock) { + return estimateWrittenPointTotalSize(tsBlock); + } + private int getCompressedSizeToReachSizeCheckpoint() { return (int) Math.max(targetChunkSize / 10, 1L); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java index 6c3e692212a8..63f4b4a7c70f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java @@ -58,6 +58,7 @@ import org.apache.tsfile.read.common.IBatchDataIterator; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.read.reader.chunk.ChunkReader; import org.apache.tsfile.read.reader.page.PageReader; @@ -154,8 +155,8 @@ public static Map> readFiles(List ts batchDataIterator.currentTime(), new TimeValuePair( batchDataIterator.currentTime(), - TsPrimitiveType.getByType( - chunkMetadata.getDataType(), batchDataIterator.currentValue()))); + Type.fromTsDataType(chunkMetadata.getDataType()) + .getTsPrimitiveType(batchDataIterator.currentValue()))); batchDataIterator.next(); } } @@ -266,8 +267,8 @@ private static void checkDataFromFrontToEnd( currTimeValuePairs.add( new TimeValuePair( batchDataIterator.currentTime(), - TsPrimitiveType.getByType( - header.getDataType(), batchDataIterator.currentValue()))); + Type.fromTsDataType(header.getDataType()) + .getTsPrimitiveType(batchDataIterator.currentValue()))); batchDataIterator.next(); } dataSize -= pageHeader.getSerializedPageSize(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java index 65621034e067..dffd4d941d05 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java @@ -63,10 +63,10 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -682,34 +682,41 @@ private TimeValuePair[] genTimeValuePair(int size, TSDataType dataType) { for (int i = 0; i < size; i++) { switch (dataType) { case BOOLEAN: - ret[i] = new TimeValuePair(rand.nextLong(), TsPrimitiveType.getByType(dataType, true)); + ret[i] = + new TimeValuePair( + rand.nextLong(), Type.fromTsDataType(dataType).getTsPrimitiveType(true)); break; case INT32: ret[i] = new TimeValuePair( - rand.nextLong(), TsPrimitiveType.getByType(dataType, rand.nextInt())); + rand.nextLong(), + Type.fromTsDataType(dataType).getTsPrimitiveType(rand.nextInt())); break; case INT64: ret[i] = new TimeValuePair( - rand.nextLong(), TsPrimitiveType.getByType(dataType, rand.nextLong())); + rand.nextLong(), + Type.fromTsDataType(dataType).getTsPrimitiveType(rand.nextLong())); break; case FLOAT: ret[i] = new TimeValuePair( - rand.nextLong(), TsPrimitiveType.getByType(dataType, rand.nextFloat())); + rand.nextLong(), + Type.fromTsDataType(dataType).getTsPrimitiveType(rand.nextFloat())); break; case DOUBLE: ret[i] = new TimeValuePair( - rand.nextLong(), TsPrimitiveType.getByType(dataType, rand.nextDouble())); + rand.nextLong(), + Type.fromTsDataType(dataType).getTsPrimitiveType(rand.nextDouble())); break; case TEXT: ret[i] = new TimeValuePair( rand.nextLong(), - TsPrimitiveType.getByType( - dataType, new Binary("a" + rand.nextDouble(), TSFileConfig.STRING_CHARSET))); + Type.fromTsDataType(dataType) + .getTsPrimitiveType( + new Binary("a" + rand.nextDouble(), TSFileConfig.STRING_CHARSET))); break; default: throw new UnSupportedDataTypeException("Unsupported data type:" + dataType); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/AscFakedSeriesReader.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/AscFakedSeriesReader.java index 574b2544d662..f4590e3560a7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/AscFakedSeriesReader.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/AscFakedSeriesReader.java @@ -20,8 +20,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.IPointReader; -import org.apache.tsfile.utils.TsPrimitiveType; import java.io.IOException; @@ -66,12 +66,14 @@ public boolean hasNextTimeValuePair() { @Override public TimeValuePair nextTimeValuePair() { if (initWithTimeList) { - return new TimeValuePair(timestamps[index++], TsPrimitiveType.getByType(DATA_TYPE, value)); + return new TimeValuePair( + timestamps[index++], Type.fromTsDataType(DATA_TYPE).getTsPrimitiveType(value)); } else { long time = startTime; startTime += interval; index++; - return new TimeValuePair(time, TsPrimitiveType.getByType(TSDataType.INT64, time % modValue)); + return new TimeValuePair( + time, Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(time % modValue)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/DescFakedSeriesReader.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/DescFakedSeriesReader.java index 2135bbe5753e..4877849873cd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/DescFakedSeriesReader.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/DescFakedSeriesReader.java @@ -20,7 +20,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.read.common.type.Type; public class DescFakedSeriesReader extends AscFakedSeriesReader { @@ -43,12 +43,14 @@ public boolean hasNextTimeValuePair() { @Override public TimeValuePair nextTimeValuePair() { if (initWithTimeList) { - return new TimeValuePair(timestamps[index--], TsPrimitiveType.getByType(DATA_TYPE, value)); + return new TimeValuePair( + timestamps[index--], Type.fromTsDataType(DATA_TYPE).getTsPrimitiveType(value)); } else { long time = startTime; startTime -= interval; index--; - return new TimeValuePair(time, TsPrimitiveType.getByType(TSDataType.INT64, time % modValue)); + return new TimeValuePair( + time, Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(time % modValue)); } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/QueryDataSetUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/QueryDataSetUtilsTest.java index ffae8fb3654e..7b65562ce616 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/QueryDataSetUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/QueryDataSetUtilsTest.java @@ -28,11 +28,16 @@ import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.column.TsBlockSerde; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; import org.junit.Test; import org.mockito.Mockito; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; @@ -47,6 +52,81 @@ public class QueryDataSetUtilsTest { private static final String BINARY_STR = "ty love zm"; + @Test + public void testReadTabletValues() throws IOException { + // Exercise the RPC length-prefixed binary format in both readers, including empty binaries. + TSDataType[] dataTypes = { + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.DATE, + TSDataType.INT64, + TSDataType.TIMESTAMP, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.BLOB, + TSDataType.STRING, + TSDataType.OBJECT + }; + Object[] expected = { + new boolean[] {true, false}, + new int[] {1, 2}, + new int[] {20260721, 20260722}, + new long[] {3L, 4L}, + new long[] {5L, 6L}, + new float[] {1.25F, 2.5F}, + new double[] {3.75D, 4.5D}, + new Binary[] { + new Binary("text-1", TSFileConfig.STRING_CHARSET), + new Binary("text-2", TSFileConfig.STRING_CHARSET) + }, + new Binary[] {Binary.EMPTY_VALUE, new Binary(new byte[] {2})}, + new Binary[] { + new Binary("string-1", TSFileConfig.STRING_CHARSET), + new Binary("string-2", TSFileConfig.STRING_CHARSET) + }, + new Binary[] { + new Binary("object-1", TSFileConfig.STRING_CHARSET), + new Binary("object-2", TSFileConfig.STRING_CHARSET) + } + }; + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try (DataOutputStream stream = new DataOutputStream(byteArrayOutputStream)) { + for (int i = 0; i < dataTypes.length; i++) { + if (expected[i] instanceof Binary[] binaries) { + for (Binary binary : binaries) { + stream.writeInt(binary.getLength()); + stream.write(binary.getValues()); + } + } else if (expected[i] instanceof int[] integers) { + // DATE is an integer in tablet RPC, not the LocalDate[] used by the Type API. + for (int value : integers) { + stream.writeInt(value); + } + } else { + Type.fromTsDataType(dataTypes[i]).serializeArray(expected[i], 2, stream); + } + } + } + byte[] serializedValues = byteArrayOutputStream.toByteArray(); + + ByteBuffer buffer = ByteBuffer.wrap(serializedValues); + assertTrue( + Arrays.deepEquals( + expected, + QueryDataSetUtils.readTabletValuesFromBuffer(buffer, dataTypes, dataTypes.length, 2))); + assertFalse(buffer.hasRemaining()); + try (DataInputStream stream = new DataInputStream(new ByteArrayInputStream(serializedValues))) { + assertTrue( + Arrays.deepEquals( + expected, + QueryDataSetUtils.readTabletValuesFromStream( + stream, dataTypes, dataTypes.length, 2))); + assertEquals(-1, stream.read()); + } + } + @Test public void testConvertTsBlockByFetchSize() throws IoTDBException, IOException { @@ -56,9 +136,32 @@ public void testConvertTsBlockByFetchSize() throws IoTDBException, IOException { compareRes(res); } + @Test + public void testConvertTsBlockListByFetchSize() throws IOException { + TSQueryDataSet result = QueryDataSetUtils.convertTsBlockByFetchSize(List.of(buildTsBlock())); + + assertEquals(Long.BYTES * 2, result.time.limit()); + assertEquals(1L, result.time.getLong()); + assertEquals(2L, result.time.getLong()); + assertEquals(1, result.valueList.size()); + assertEquals(1, result.bitmapList.size()); + assertEquals(Byte.BYTES, result.valueList.get(0).limit()); + assertEquals((byte) 1, result.valueList.get(0).get()); + assertEquals((byte) 0x80, result.bitmapList.get(0).get()); + } + private IQueryExecution buildQueryExecution() throws IoTDBException, IOException { IQueryExecution queryExecution = Mockito.mock(IQueryExecution.class); Mockito.when(queryExecution.getOutputValueColumnCount()).thenReturn(6); + TsBlock tsBlock = buildTsBlock(); + Mockito.when(queryExecution.getBatchResult()) + .thenReturn(Optional.of(tsBlock), Optional.empty()); + Mockito.when(queryExecution.getByteBufferBatchResult()) + .thenReturn(Optional.of(new TsBlockSerde().serialize(tsBlock)), Optional.empty()); + return queryExecution; + } + + private TsBlock buildTsBlock() { TsBlockBuilder builder = new TsBlockBuilder( Arrays.asList( @@ -84,11 +187,7 @@ private IQueryExecution buildQueryExecution() throws IoTDBException, IOException builder.getColumnBuilder(4).writeDouble(3.14d); builder.getColumnBuilder(5).appendNull(); builder.declarePosition(); - Mockito.when(queryExecution.getBatchResult()) - .thenReturn(Optional.of(builder.build()), Optional.empty()); - Mockito.when(queryExecution.getByteBufferBatchResult()) - .thenReturn(Optional.of(new TsBlockSerde().serialize(builder.build())), Optional.empty()); - return queryExecution; + return builder.build(); } private void compareRes(Pair res) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SerializeUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SerializeUtilsTest.java index c854c232424c..38cc89f2ea0c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SerializeUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SerializeUtilsTest.java @@ -26,6 +26,7 @@ import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.DescReadBatchData; import org.apache.tsfile.read.common.DescReadWriteBatchData; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.TsPrimitiveType; @@ -205,22 +206,55 @@ public void serdesTEXTBatchDataTest() { } } + /** DATE and TIMESTAMP use the INT32 and INT64 wire encodings while retaining their type tags. */ + @Test + public void serdesDateAndTimestampBatchDataTest() { + BatchData dateData = new BatchData(TSDataType.DATE); + BatchData timestampData = new BatchData(TSDataType.TIMESTAMP); + for (int i = 0; i < 3; i++) { + dateData.putInt(i, 2024 + i); + timestampData.putLong(i, 1_000L + i); + } + + ByteArrayOutputStream dateBytes = new ByteArrayOutputStream(); + SerializeUtils.serializeBatchData(dateData, new DataOutputStream(dateBytes)); + BatchData deserializedDate = + SerializeUtils.deserializeBatchData(ByteBuffer.wrap(dateBytes.toByteArray())); + ByteArrayOutputStream timestampBytes = new ByteArrayOutputStream(); + SerializeUtils.serializeBatchData(timestampData, new DataOutputStream(timestampBytes)); + BatchData deserializedTimestamp = + SerializeUtils.deserializeBatchData(ByteBuffer.wrap(timestampBytes.toByteArray())); + + Assert.assertEquals(TSDataType.DATE, deserializedDate.getDataType()); + Assert.assertEquals(TSDataType.TIMESTAMP, deserializedTimestamp.getDataType()); + for (int i = 0; i < 3; i++) { + Assert.assertEquals(2024 + i, deserializedDate.getIntByIndex(i)); + Assert.assertEquals(1_000L + i, deserializedTimestamp.getLongByIndex(i)); + } + } + /** This method tests SerializeUtils.serializeTVPair() and SerializeUtils.deserializeTVPair() */ @Test public void serdesTVPairTest() { List TVPairs = new ArrayList<>(); - TimeValuePair p1 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.BOOLEAN, true)); + TimeValuePair p1 = + new TimeValuePair(0, Type.fromTsDataType(TSDataType.BOOLEAN).getTsPrimitiveType(true)); TVPairs.add(p1); - TimeValuePair p2 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.INT32, 1)); + TimeValuePair p2 = + new TimeValuePair(0, Type.fromTsDataType(TSDataType.INT32).getTsPrimitiveType(1)); TVPairs.add(p2); - TimeValuePair p3 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.INT64, 1L)); + TimeValuePair p3 = + new TimeValuePair(0, Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(1L)); TVPairs.add(p3); - TimeValuePair p4 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.FLOAT, 1.0f)); + TimeValuePair p4 = + new TimeValuePair(0, Type.fromTsDataType(TSDataType.FLOAT).getTsPrimitiveType(1.0f)); TVPairs.add(p4); - TimeValuePair p5 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.DOUBLE, 1.0d)); + TimeValuePair p5 = + new TimeValuePair(0, Type.fromTsDataType(TSDataType.DOUBLE).getTsPrimitiveType(1.0d)); TVPairs.add(p5); TimeValuePair p6 = - new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.TEXT, BytesUtils.valueOf("a"))); + new TimeValuePair( + 0, Type.fromTsDataType(TSDataType.TEXT).getTsPrimitiveType(BytesUtils.valueOf("a"))); TVPairs.add(p6); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -233,22 +267,66 @@ public void serdesTVPairTest() { } } + /** + * Covers the DATE/TIMESTAMP aliases and binary type aliases in TVPair serialization. The + * Long.MIN_VALUE timestamp is the wire-format null-value marker and must not consume a value. + */ + @Test + public void serdesAdditionalTVPairTypesTest() { + List pairs = + Arrays.asList( + new TimeValuePair(1, Type.fromTsDataType(TSDataType.DATE).getTsPrimitiveType(2024)), + new TimeValuePair(2, Type.fromTsDataType(TSDataType.TIMESTAMP).getTsPrimitiveType(2L)), + new TimeValuePair( + 3, + Type.fromTsDataType(TSDataType.BLOB) + .getTsPrimitiveType(BytesUtils.valueOf("blob"))), + new TimeValuePair( + 4, + Type.fromTsDataType(TSDataType.STRING) + .getTsPrimitiveType(BytesUtils.valueOf("string")))); + + for (TimeValuePair pair : pairs) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializeUtils.serializeTVPair(pair, new DataOutputStream(baos)); + Assert.assertEquals( + pair, SerializeUtils.deserializeTVPair(ByteBuffer.wrap(baos.toByteArray()))); + } + + // A sentinel timestamp represents a null value; serialization writes no value bytes after it. + TimeValuePair nullValue = + new TimeValuePair( + Long.MIN_VALUE, Type.fromTsDataType(TSDataType.INT32).getTsPrimitiveType(42)); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializeUtils.serializeTVPair(nullValue, new DataOutputStream(baos)); + TimeValuePair deserialized = + SerializeUtils.deserializeTVPair(ByteBuffer.wrap(baos.toByteArray())); + Assert.assertEquals(Long.MIN_VALUE, deserialized.getTimestamp()); + Assert.assertNull(deserialized.getValue()); + } + /** This method tests SerializeUtils.serializeTVPairs() and SerializeUtils.deserializeTVPairs() */ @Test public void serdesTVPairsTest() { List> TVPairs = new ArrayList<>(); - TimeValuePair p1 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.BOOLEAN, true)); + TimeValuePair p1 = + new TimeValuePair(0, Type.fromTsDataType(TSDataType.BOOLEAN).getTsPrimitiveType(true)); TVPairs.add(Collections.singletonList(p1)); - TimeValuePair p2 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.INT32, 1)); + TimeValuePair p2 = + new TimeValuePair(0, Type.fromTsDataType(TSDataType.INT32).getTsPrimitiveType(1)); TVPairs.add(Collections.singletonList(p2)); - TimeValuePair p3 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.INT64, 1L)); + TimeValuePair p3 = + new TimeValuePair(0, Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(1L)); TVPairs.add(Collections.singletonList(p3)); - TimeValuePair p4 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.FLOAT, 1.0f)); + TimeValuePair p4 = + new TimeValuePair(0, Type.fromTsDataType(TSDataType.FLOAT).getTsPrimitiveType(1.0f)); TVPairs.add(Collections.singletonList(p4)); - TimeValuePair p5 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.DOUBLE, 1.0d)); + TimeValuePair p5 = + new TimeValuePair(0, Type.fromTsDataType(TSDataType.DOUBLE).getTsPrimitiveType(1.0d)); TVPairs.add(Collections.singletonList(p5)); TimeValuePair p6 = - new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.TEXT, BytesUtils.valueOf("a"))); + new TimeValuePair( + 0, Type.fromTsDataType(TSDataType.TEXT).getTsPrimitiveType(BytesUtils.valueOf("a"))); TVPairs.add(Collections.singletonList(p6)); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -261,6 +339,36 @@ public void serdesTVPairsTest() { } } + /** Ensures the batch TVPair dispatcher uses the same primitive encodings as single pairs. */ + @Test + public void serdesAdditionalTVPairsTypesTest() { + List> pairs = + Arrays.asList( + Collections.singletonList( + new TimeValuePair( + 1, Type.fromTsDataType(TSDataType.DATE).getTsPrimitiveType(2024))), + Collections.singletonList( + new TimeValuePair( + 2, Type.fromTsDataType(TSDataType.TIMESTAMP).getTsPrimitiveType(2L))), + Collections.singletonList( + new TimeValuePair( + 3, + Type.fromTsDataType(TSDataType.BLOB) + .getTsPrimitiveType(BytesUtils.valueOf("blob")))), + Collections.singletonList( + new TimeValuePair( + 4, + Type.fromTsDataType(TSDataType.STRING) + .getTsPrimitiveType(BytesUtils.valueOf("string"))))); + + for (List pair : pairs) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializeUtils.serializeTVPairs(pair, new DataOutputStream(baos)); + Assert.assertEquals( + pair, SerializeUtils.deserializeTVPairs(ByteBuffer.wrap(baos.toByteArray()))); + } + } + /** This method tests SerializeUtils.serializeObject() and SerializeUtils.deserializeObject() */ @Test public void serdesObjectTest() { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchArchitectureTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchArchitectureTest.java new file mode 100644 index 000000000000..543002ecd365 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchArchitectureTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import com.tngtech.archunit.core.importer.ClassFileImporter; +import com.tngtech.archunit.core.importer.ImportOption; +import org.junit.Test; + +public class TSDataTypeSwitchArchitectureTest { + + @Test + public void datanodeProductionCodeMustUseTypeService() { + // This module-level check covers every DataNode production class in the datanode artifact. + // Do not freeze violations or swallow failures: existing violations must fail this test too. + TSDataTypeSwitchRule.RULE.check( + new ClassFileImporter() + .withImportOption(new ImportOption.DoNotIncludeTests()) + // Avoid building the entire server dependency graph: only switch owners need checking. + .withImportOption(location -> TSDataTypeSwitchRule.hasSwitch(location.asURI())) + .importPackages("org.apache.iotdb.db")); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchRule.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchRule.java new file mode 100644 index 000000000000..6dad4d745e8b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchRule.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import com.tngtech.archunit.core.domain.JavaClass; +import com.tngtech.archunit.core.domain.JavaModifier; +import com.tngtech.archunit.lang.ArchCondition; +import com.tngtech.archunit.lang.ArchRule; +import com.tngtech.archunit.lang.ConditionEvents; +import com.tngtech.archunit.lang.SimpleConditionEvent; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.classes; + +final class TSDataTypeSwitchRule { + private static final String SWITCH_MAP = + "$SwitchMap$" + TSDataType.class.getName().replace('.', '$'); + private static final String SERVICE_DESCRIPTOR = Type.getDescriptor(TypeService.class); + + static final ArchRule RULE = + classes() + .should( + new ArchCondition<>("only switch on TSDataType inside TypeService") { + @Override + public void check(JavaClass javaClass, ConditionEvents events) { + if (javaClass.getModifiers().contains(JavaModifier.SYNTHETIC) + || javaClass.isAssignableTo(TypeService.class)) { + return; + } + SwitchVisitor visitor = read(javaClass.getSource().orElseThrow().getUri()); + for (SwitchAccess access : visitor.switches) { + if (!visitor.serviceMethods.contains(access.method)) { + events.add( + SimpleConditionEvent.violated( + javaClass, + javaClass.getName() + + "." + + access.method + + " switches on TSDataType outside TypeService at (" + + visitor.sourceFile + + ":" + + access.line + + ")")); + } + } + } + }) + .allowEmptyShould(true); + + private TSDataTypeSwitchRule() {} + + static boolean hasSwitch(URI uri) { + return !read(uri).switches.isEmpty(); + } + + private static SwitchVisitor read(URI uri) { + try (InputStream input = uri.toURL().openStream()) { + SwitchVisitor visitor = new SwitchVisitor(); + new ClassReader(input).accept(visitor, ClassReader.SKIP_FRAMES); + return visitor; + } catch (IOException e) { + throw new UncheckedIOException("Cannot inspect " + uri, e); + } + } + + // ArchUnit deliberately drops $SwitchMap$ accesses. Read them with ASM instead of treating + // every ordinal() invocation as a switch, which would incorrectly reject ordinary enum usage. + private static final class SwitchVisitor extends ClassVisitor { + private final List switches = new ArrayList<>(); + private final Set serviceMethods = new HashSet<>(); + private String owner; + private String sourceFile; + private boolean synthetic; + + private SwitchVisitor() { + super(Opcodes.ASM9); + } + + @Override + public void visit( + int version, + int access, + String name, + String signature, + String superName, + String[] interfaces) { + owner = name; + synthetic = (access & Opcodes.ACC_SYNTHETIC) != 0; + } + + @Override + public void visitSource(String source, String debug) { + sourceFile = source; + } + + @Override + public MethodVisitor visitMethod( + int access, String name, String descriptor, String signature, String[] exceptions) { + if (synthetic) { + // The compiler's mapping-array initializer is not a source-level switch. + return null; + } + return new MethodVisitor(Opcodes.ASM9) { + private int line; + + @Override + public void visitLineNumber(int lineNumber, Label start) { + line = lineNumber; + } + + @Override + public void visitFieldInsn( + int opcode, String fieldOwner, String fieldName, String fieldType) { + // The repository compiles with Java 17 javac, for both switch statements and expressions. + if (opcode == Opcodes.GETSTATIC + && fieldName.equals(SWITCH_MAP) + && fieldType.equals("[I")) { + switches.add(new SwitchAccess(name + descriptor, line)); + } + } + + @Override + public void visitInvokeDynamicInsn( + String name, String descriptor, Handle bootstrap, Object... arguments) { + // Only the lambda implementing TypeService is exempt, not all methods of its holder. + if (bootstrap.getOwner().equals("java/lang/invoke/LambdaMetafactory") + && Type.getReturnType(descriptor).getDescriptor().equals(SERVICE_DESCRIPTOR) + && arguments.length > 1 + && arguments[1] instanceof Handle implementation + && implementation.getOwner().equals(owner) + && implementation.getName().startsWith("lambda$")) { + serviceMethods.add(implementation.getName() + implementation.getDesc()); + } + } + }; + } + } + + private static final class SwitchAccess { + private final String method; + private final int line; + + private SwitchAccess(String method, int line) { + this.method = method; + this.line = line; + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchRuleTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchRuleTest.java new file mode 100644 index 000000000000..53a0f2b41084 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TSDataTypeSwitchRuleTest.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import com.tngtech.archunit.core.importer.ClassFileImporter; +import com.tngtech.archunit.lang.EvaluationResult; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.junit.Test; + +import java.util.List; +import java.util.function.ToIntFunction; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TSDataTypeSwitchRuleTest { + + @Test + public void detectsStatementAndExpression() { + // Both javac switch forms must report the business method and a nonzero source line. + List violations = violations(Switches.class); + assertEquals(2, violations.size()); + assertTrue(violations.stream().anyMatch(message -> message.contains(".statement("))); + assertTrue(violations.stream().anyMatch(message -> message.contains(".expression("))); + for (String message : violations) { + assertTrue(message, message.matches(".*TSDataTypeSwitchRuleTest.java:[1-9][0-9]*.*")); + } + } + + @Test + public void detectsLambdaAndAnonymousClass() { + // Switches cannot escape detection by being moved into lambdas or anonymous classes. + assertEquals(1, violations(LambdaSwitch.class).size()); + assertEquals(1, violations(AnonymousSwitch.VALUE.getClass()).size()); + } + + @Test + public void allowsOtherEnumsAndOrdinal() { + // An ordinal read beside an unrelated enum switch is legal, as is reading an enum constant. + assertTrue(violations(OrdinaryUsage.class).isEmpty()); + } + + @Test + public void allowsTypeServiceImplementationAndRegisteredHolder() { + // Actual TypeService implementations and the existing holder are valid dispatch locations. + assertTrue(violations(SwitchService.class, TypeServices.class).isEmpty()); + } + + @Test + public void doesNotExemptSimilarlyNamedBusinessClass() { + // A class name containing TypeService is not sufficient to bypass the rule. + assertFalse(violations(NotATypeService.class).isEmpty()); + } + + private static List violations(Class... classes) { + EvaluationResult result = + TSDataTypeSwitchRule.RULE.evaluate(new ClassFileImporter().importClasses(classes)); + return result.getFailureReport().getDetails(); + } + + @Test + public void onlyExemptsServiceLambdaInsideHolder() { + List violations = violations(MixedHolder.class); + assertEquals(2, violations.size()); + assertTrue(violations.stream().anyMatch(message -> message.contains(".business("))); + } + + static class Switches { + int statement(TSDataType type) { + switch (type) { + case INT32: + return 1; + default: + return 0; + } + } + + int expression(TSDataType type) { + return switch (type) { + case DOUBLE -> 2; + default -> 0; + }; + } + } + + static class LambdaSwitch { + ToIntFunction value() { + return type -> + switch (type) { + case FLOAT -> 3; + default -> 0; + }; + } + } + + static class AnonymousSwitch { + static final ToIntFunction VALUE = + new ToIntFunction() { + @Override + public int applyAsInt(TSDataType type) { + return switch (type) { + case INT64 -> 4; + default -> 0; + }; + } + }; + } + + static class OrdinaryUsage { + int value(TSDataType type, Thread.State state) { + int ordinal = type.ordinal(); + int stateValue = + switch (state) { + case NEW -> 1; + default -> 0; + }; + return ordinal + stateValue + TSDataType.INT32.ordinal(); + } + } + + static class SwitchService implements TypeService { + @Override + public Integer call(Type type) { + return dispatch(TSDataType.INT32); + } + + int dispatch(TSDataType type) { + return switch (type) { + case INT32 -> 1; + default -> 0; + }; + } + } + + static class NotATypeService { + int dispatch(TSDataType type) { + return switch (type) { + case INT32 -> 1; + default -> 0; + }; + } + } + + static class MixedHolder { + static final TypeService SERVICE = + type -> + switch (TSDataType.INT32) { + case INT32 -> 1; + default -> 0; + }; + + static final ToIntFunction ORDINARY = + type -> + switch (type) { + case INT32 -> 1; + default -> 0; + }; + + int business(TSDataType type) { + return switch (type) { + case INT32 -> 1; + default -> 0; + }; + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TypeServicesTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TypeServicesTest.java new file mode 100644 index 000000000000..6afb127905bd --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TypeServicesTest.java @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BinaryLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Literal; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; +import org.apache.tsfile.read.common.block.column.FloatColumn; +import org.apache.tsfile.read.common.block.column.IntColumn; +import org.apache.tsfile.read.common.block.column.LongColumn; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.util.Optional; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class TypeServicesTest { + + @Test + public void testTabletBinarySerializationMatchesLegacyFormat() throws IOException { + Binary[] values = { + new Binary(new byte[] {1, 2, 3}), + null, + new Binary((byte[]) null), + new Binary(new byte[0]), + new Binary(new byte[100]) + }; + // Four active rows: length + payload, then three zero-length placeholders. No presence bytes. + byte[] expected = + ByteBuffer.allocate(19) + .putInt(3) + .put(new byte[] {1, 2, 3}) + .putInt(0) + .putInt(0) + .putInt(0) + .array(); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type type = Type.fromTsDataType(dataType); + int size = + TypeServices.StorageEngine.INSERT_TABLET_SERIALIZED_COLUMN_SIZE_SERVICE + .call(type) + .size(values, 4); + assertEquals(expected.length, size); + ByteBuffer buffer = ByteBuffer.allocate(size); + TypeServices.StorageEngine.RAW_ARRAY_BYTE_BUFFER_SERIALIZER_SERVICE + .call(type) + .serialize(values, 4, buffer); + assertEquals(size, buffer.position()); + assertArrayEquals(expected, buffer.array()); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream stream = new DataOutputStream(bytes)) { + TypeServices.StorageEngine.RAW_ARRAY_OUTPUT_STREAM_SERIALIZER_SERVICE + .call(type) + .serialize(values, 4, stream); + } + assertArrayEquals(expected, bytes.toByteArray()); + } + } + + @Test + public void testWalBinaryRangeSizeMatchesWrittenBytes() { + Binary[] values = { + new Binary(new byte[100]), + new Binary(new byte[] {1, 2, 3}), + null, + new Binary((byte[]) null), + new Binary(new byte[0]), + new Binary(new byte[100]) + }; + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type type = Type.fromTsDataType(dataType); + int size = + TypeServices.StorageEngine.INSERT_TABLET_SERIALIZED_COLUMN_SIZE_SERVICE + .call(type) + .size(values, 1, 5); + assertEquals(19, size); + ByteBuffer buffer = ByteBuffer.allocate(size); + TypeServices.StorageEngine.WAL_ARRAY_WRITER_SERVICE + .call(type) + .write(values, new WALByteBufferForTest(buffer), 1, 5); + assertEquals(size, buffer.position()); + buffer.flip(); + assertEquals(3, buffer.getInt()); + byte[] payload = new byte[3]; + buffer.get(payload); + assertArrayEquals(new byte[] {1, 2, 3}, payload); + assertEquals(0, buffer.getInt()); + assertEquals(0, buffer.getInt()); + assertEquals(0, buffer.getInt()); + assertFalse(buffer.hasRemaining()); + } + } + + @Test + public void testMaxMinByReadsXWithoutModifyingInput() { + // A selected timestamp must replace the result, including on reuse, without changing the input. + for (TSDataType dataType : new TSDataType[] {TSDataType.INT64, TSDataType.TIMESTAMP}) { + Type type = Type.fromTsDataType(dataType); + Column input = new LongColumn(2, Optional.empty(), new long[] {2499, 8499}); + TsPrimitiveType result = type.getTsPrimitiveType(); + TypeServices.Aggregation.MaxMinByAccumulatorStrategy strategy = + TypeServices.Aggregation.MAX_MIN_BY_ACCUMULATOR_STRATEGY_SERVICE.call(type); + strategy.setXResult(result, input, 1); + assertEquals(8499, result.getLong()); + strategy.setXResult(result, input, 0); + assertEquals(2499, result.getLong()); + assertEquals(2499, input.getLong(0)); + assertEquals(8499, input.getLong(1)); + } + } + + @Test + public void testInsertSerializedSizesPreserveNullBinaryAndActiveRows() { + Binary value = new Binary(new byte[] {1, 2, 3}); + Binary[] values = {value, null, new Binary((byte[]) null), new Binary(new byte[100])}; + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type type = Type.fromTsDataType(dataType); + assertEquals( + 7, + TypeServices.StorageEngine.INSERT_ROW_SERIALIZED_VALUE_SIZE_SERVICE + .call(type) + .applyAsInt(value)); + // The spare capacity must not contribute to the serialized size. + assertEquals( + 15, + TypeServices.StorageEngine.INSERT_TABLET_SERIALIZED_COLUMN_SIZE_SERVICE + .call(type) + .size(values, 3)); + } + assertEquals( + 12, + TypeServices.StorageEngine.INSERT_TABLET_SERIALIZED_COLUMN_SIZE_SERVICE + .call(Type.fromTsDataType(TSDataType.DATE)) + .size(new int[5], 3)); + assertEquals( + 24, + TypeServices.StorageEngine.INSERT_TABLET_SERIALIZED_COLUMN_SIZE_SERVICE + .call(Type.fromTsDataType(TSDataType.TIMESTAMP)) + .size(new long[5], 3)); + } + + @Test + public void testPlainFastPathExcludesObject() { + for (TSDataType type : + new TSDataType[] { + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.DATE, + TSDataType.INT64, + TSDataType.TIMESTAMP, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB + }) { + assertTrue( + TypeServices.StorageEngine.TABLET_PLAIN_FAST_PATH_SERVICE.call( + Type.fromTsDataType(type))); + } + assertFalse( + TypeServices.StorageEngine.TABLET_PLAIN_FAST_PATH_SERVICE.call( + Type.fromTsDataType(TSDataType.OBJECT))); + } + + @Test + public void testAlteredDataTypeNumericConversion() { + final Column intSource = + new IntColumn( + 3, + Optional.of(new boolean[] {false, true, false}), + new int[] {1, 0, -2}, + TSDataType.INT32); + final Column longSource = new LongColumn(1, Optional.of(new boolean[] {false}), new long[] {3}); + final Column floatSource = + new FloatColumn(1, Optional.of(new boolean[] {false}), new float[] {4.5F}); + + final Column int64Result = transform(intSource, TSDataType.INT64); + final Column floatResult = transform(intSource, TSDataType.FLOAT); + final Column doubleResult = transform(intSource, TSDataType.DOUBLE); + + assertEquals(1L, int64Result.getLong(0)); + assertTrue(int64Result.isNull(1)); + assertEquals(-2L, int64Result.getLong(2)); + assertEquals(1.0F, floatResult.getFloat(0), 0.0F); + assertTrue(floatResult.isNull(1)); + assertEquals(-2.0F, floatResult.getFloat(2), 0.0F); + assertEquals(1.0, doubleResult.getDouble(0), 0.0); + assertTrue(doubleResult.isNull(1)); + assertEquals(-2.0, doubleResult.getDouble(2), 0.0); + assertEquals(3.0, transform(longSource, TSDataType.DOUBLE).getDouble(0), 0.0); + assertEquals(4.5, transform(floatSource, TSDataType.DOUBLE).getDouble(0), 0.0); + } + + @Test + public void testAlteredDataTypeTextConversion() { + final int date = DateUtils.parseDateExpressionToInt(LocalDate.of(2026, 8, 12)); + final Column dateColumn = + new IntColumn(1, Optional.of(new boolean[] {false}), new int[] {date}, TSDataType.DATE); + final Column booleanColumn = + new BooleanColumn(1, Optional.of(new boolean[] {false}), new boolean[] {true}); + + assertEquals("2026-08-12", transform(dateColumn, TSDataType.TEXT).getBinary(0).toString()); + assertEquals("true", transform(booleanColumn, TSDataType.STRING).getBinary(0).toString()); + } + + @Test + public void testAlteredDataTypeIncompatibleValuesBecomeNull() { + final Column source = + new BinaryColumn( + 2, + Optional.of(new boolean[] {false, false}), + new Binary[] { + new Binary("1", StandardCharsets.UTF_8), new Binary("2", StandardCharsets.UTF_8) + }); + + final Column result = transform(source, TSDataType.INT32); + + assertTrue(result.isNull(0)); + assertTrue(result.isNull(1)); + } + + @Test + public void testAlteredDataTypeCompatibleColumnIsReused() { + final Column intSource = + new IntColumn(1, Optional.of(new boolean[] {false}), new int[] {1}, TSDataType.INT32); + final Column dateSource = + new IntColumn(1, Optional.of(new boolean[] {false}), new int[] {1}, TSDataType.DATE); + + final Column sameTypeResult = transform(intSource, TSDataType.INT32); + final Column intToDateResult = transform(intSource, TSDataType.DATE); + final Column dateToIntResult = transform(dateSource, TSDataType.INT32); + + assertSame(intSource, sameTypeResult); + assertSame(intSource, intToDateResult); + assertFalse(sameTypeResult.isNull(0)); + assertTrue(dateToIntResult.isNull(0)); + } + + @Test + public void testPreparedParameterLiteralConversion() { + final Pair stringResult = convertPreparedParameter(TSDataType.STRING, "it's"); + final Pair blobResult = + convertPreparedParameter(TSDataType.BLOB, new byte[] {(byte) 0xAB, 0x01}); + + assertEquals("it's", ((StringLiteral) stringResult.left).getValue()); + assertEquals("'it''s'", stringResult.right); + assertEquals("AB01", ((BinaryLiteral) blobResult.left).toHexString()); + assertEquals("X'AB01'", blobResult.right); + } + + private static Column transform(final Column source, final TSDataType targetType) { + return TypeServices.Transformation.ALTERED_DATA_TYPE_COLUMN_TRANSFORMER_SERVICE + .call(Type.fromTsDataType(targetType)) + .transform(source, source.getPositionCount()); + } + + private static Pair convertPreparedParameter( + final TSDataType type, final Object value) { + return TypeServices.ValueConversion.PREPARED_PARAMETER_LITERAL_SERVICE + .call(Type.fromTsDataType(type)) + .apply(value); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java index 708f2113cdf2..6b5e93eea50e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java @@ -47,6 +47,22 @@ public class AlignedTVListTest { + @Test + public void testExtendColumnRetainsImplicitNullsOnFirstWrite() { + AlignedTVList list = + AlignedTVList.newAlignedList(new ArrayList<>(Collections.singletonList(TSDataType.INT64))); + list.putAlignedValue(0, new Object[] {null}); + list.putAlignedValue(1, new Object[] {1L}); + list.extendColumn(TSDataType.INT32); + Assert.assertNull(list.getValues().get(1).get(0)); + list.putAlignedValue(2, new Object[] {2L, 7}); + Assert.assertTrue(list.isNullValue(0, 0)); + Assert.assertTrue(list.isNullValue(0, 1)); + Assert.assertTrue(list.isNullValue(1, 1)); + Assert.assertEquals(7, list.getIntByValueIndex(2, 1)); + list.clear(); + } + @Test public void testValueListArrayMemCostExcludesLazyBitmapAndNullPlaceholder() { long expected = (long) ARRAY_SIZE * Long.BYTES + NUM_BYTES_ARRAY_HEADER + NUM_BYTES_OBJECT_REF; @@ -513,6 +529,7 @@ public void testAllValueDeletedMapMatchesPerColumnNullState() { @Test public void testNullPrimitiveArrayCanBeClonedAndSerialized() throws IOException { + // Entirely null blocks must remain lazy during cloning and retain WAL placeholders. AlignedTVList tvList = AlignedTVList.newAlignedList(Arrays.asList(TSDataType.TEXT, TSDataType.INT64)); for (int i = 0; i <= ARRAY_SIZE; i++) { @@ -530,6 +547,7 @@ public void testNullPrimitiveArrayCanBeClonedAndSerialized() throws IOException WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.allocate(tvList.serializedSize())); tvList.serializeToWAL(walBuffer); + Assert.assertEquals(tvList.serializedSize(), walBuffer.getBuffer().position()); AlignedTVList deserializedTvList = AlignedTVList.deserialize( new DataInputStream(new ByteArrayInputStream(walBuffer.getBuffer().array()))); @@ -540,6 +558,53 @@ public void testNullPrimitiveArrayCanBeClonedAndSerialized() throws IOException "[value, 1]", deserializedTvList.getAlignedValue(ARRAY_SIZE + 1).toString()); } + @Test + public void testNullBlocksSerializeForAllScalarTypes() throws IOException { + // Cover every WAL scalar writer with an unmaterialized block and a later real value. + List dataTypes = + Arrays.asList( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.DATE, + TSDataType.INT64, + TSDataType.TIMESTAMP, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB, + TSDataType.OBJECT); + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + Object[] rowValues = {true, 7, 20260908, 9L, 10L, 1.25F, 2.5D, binary, binary, binary, binary}; + AlignedTVList list = AlignedTVList.newAlignedList(dataTypes); + for (int row = 0; row <= ARRAY_SIZE; row++) { + list.putAlignedValue(row, new Object[dataTypes.size()]); + } + list.putAlignedValue(ARRAY_SIZE + 1L, rowValues); + AlignedTVList cloned = list.clone(); + for (int column = 0; column < dataTypes.size(); column++) { + Assert.assertNull(cloned.getValues().get(column).get(0)); + } + + WALByteBufferForTest buffer = + new WALByteBufferForTest(ByteBuffer.allocate(list.serializedSize())); + list.serializeToWAL(buffer); + Assert.assertEquals(list.serializedSize(), buffer.getBuffer().position()); + DataInputStream stream = + new DataInputStream(new ByteArrayInputStream(buffer.getBuffer().array())); + AlignedTVList restored = AlignedTVList.deserialize(stream); + Assert.assertEquals(-1, stream.read()); + Assert.assertEquals(list.rowCount(), restored.rowCount()); + for (int row = 0; row <= ARRAY_SIZE; row++) { + for (int column = 0; column < dataTypes.size(); column++) { + Assert.assertTrue(restored.isNullValue(row, column)); + } + } + Assert.assertEquals( + list.getAlignedValue(ARRAY_SIZE + 1).toString(), + restored.getAlignedValue(ARRAY_SIZE + 1).toString()); + } + @Test public void testClone() { List dataTypes = new ArrayList<>(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/type/InternalTypeManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/type/InternalTypeManager.java index c3d2fcd880fc..8d236c888fa4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/type/InternalTypeManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/type/InternalTypeManager.java @@ -82,66 +82,21 @@ public static TSDataType getTSDataType(Type type) { if (type == null) { return null; } - TypeEnum typeEnum = type.getTypeEnum(); - switch (typeEnum) { - case TEXT: - return TSDataType.TEXT; - case FLOAT: - return TSDataType.FLOAT; - case DOUBLE: - return TSDataType.DOUBLE; - case INT32: - return TSDataType.INT32; - case INT64: - return TSDataType.INT64; - case BOOLEAN: - return TSDataType.BOOLEAN; - case UNKNOWN: - return TSDataType.UNKNOWN; - case DATE: - return TSDataType.DATE; - case TIMESTAMP: - return TSDataType.TIMESTAMP; - case BLOB: - case ROW: - return TSDataType.BLOB; - case STRING: - return TSDataType.STRING; - case OBJECT: - return TSDataType.OBJECT; - default: - throw new IllegalArgumentException(); + final TypeEnum typeEnum = type.getTypeEnum(); + // TSDataType has no ROW counterpart, so preserve its existing binary representation. + if (typeEnum == TypeEnum.ROW) { + return TSDataType.BLOB; } + if (typeEnum == TypeEnum.VECTOR) { + throw new IllegalArgumentException(); + } + return TSDataType.valueOf(typeEnum.name()); } public static Type fromTSDataType(TSDataType dataType) { - switch (dataType) { - case TEXT: - return TEXT; - case FLOAT: - return FLOAT; - case DOUBLE: - return DOUBLE; - case INT32: - return INT32; - case INT64: - return INT64; - case BOOLEAN: - return BOOLEAN; - case UNKNOWN: - return UNKNOWN; - case DATE: - return DATE; - case TIMESTAMP: - return TIMESTAMP; - case BLOB: - return BLOB; - case OBJECT: - return OBJECT; - case STRING: - return STRING; - default: - throw new IllegalArgumentException(); + if (dataType == TSDataType.VECTOR) { + throw new IllegalArgumentException(); } + return Type.fromTsDataType(dataType); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/TypeServices.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/TypeServices.java new file mode 100644 index 000000000000..b4c8333a3fe7 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/TypeServices.java @@ -0,0 +1,671 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.commons.udf.builtin; + +import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; +import org.apache.iotdb.udf.api.access.Row; +import org.apache.iotdb.udf.api.access.RowWindow; +import org.apache.iotdb.udf.api.collector.PointCollector; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; +import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.service.TypeService; + +import java.io.IOException; + +/** Type-specific operations used by numeric built-in UDFs. */ +final class TypeServices { + + // Select the primitive Row accessor once per UDF instance instead of switching for every row. + static final TypeService VALUE_TREND_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> (target, row) -> target.previousInt = row.getInt(0); + case INT64 -> (target, row) -> target.previousLong = row.getLong(0); + case FLOAT -> (target, row) -> target.previousFloat = row.getFloat(0); + case DOUBLE -> (target, row) -> target.previousDouble = row.getDouble(0); + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (target, row) -> { + throw target.invalidDataType(); + }; + }; + + static final TypeService VALUE_DIFFERENCE_OPERATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (target, time, row, collector) -> { + int current = row.getInt(0); + collector.putInt(time, current - target.previousInt); + target.previousInt = current; + }; + case INT64 -> + (target, time, row, collector) -> { + long current = row.getLong(0); + collector.putLong(time, current - target.previousLong); + target.previousLong = current; + }; + case FLOAT -> + (target, time, row, collector) -> { + float current = row.getFloat(0); + collector.putFloat(time, current - target.previousFloat); + target.previousFloat = current; + }; + case DOUBLE -> + (target, time, row, collector) -> { + double current = row.getDouble(0); + collector.putDouble(time, current - target.previousDouble); + target.previousDouble = current; + }; + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (target, time, row, collector) -> { + throw target.invalidDataType(); + }; + }; + + static final TypeService NON_NEGATIVE_VALUE_DIFFERENCE_OPERATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (target, time, row, collector) -> { + int current = row.getInt(0); + collector.putInt(time, Math.abs(current - target.previousInt)); + target.previousInt = current; + }; + case INT64 -> + (target, time, row, collector) -> { + long current = row.getLong(0); + collector.putLong(time, Math.abs(current - target.previousLong)); + target.previousLong = current; + }; + case FLOAT -> + (target, time, row, collector) -> { + float current = row.getFloat(0); + collector.putFloat(time, Math.abs(current - target.previousFloat)); + target.previousFloat = current; + }; + case DOUBLE -> + (target, time, row, collector) -> { + double current = row.getDouble(0); + collector.putDouble(time, Math.abs(current - target.previousDouble)); + target.previousDouble = current; + }; + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (target, time, row, collector) -> { + throw target.invalidDataType(); + }; + }; + + static final TypeService DERIVATIVE_OPERATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (target, time, row, collector, timeDelta) -> { + int current = row.getInt(0); + collector.putDouble(time, (current - target.previousInt) / timeDelta); + target.previousInt = current; + }; + case INT64 -> + (target, time, row, collector, timeDelta) -> { + long current = row.getLong(0); + collector.putDouble(time, (current - target.previousLong) / timeDelta); + target.previousLong = current; + }; + case FLOAT -> + (target, time, row, collector, timeDelta) -> { + float current = row.getFloat(0); + collector.putDouble(time, (current - target.previousFloat) / timeDelta); + target.previousFloat = current; + }; + case DOUBLE -> + (target, time, row, collector, timeDelta) -> { + double current = row.getDouble(0); + collector.putDouble(time, (current - target.previousDouble) / timeDelta); + target.previousDouble = current; + }; + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (target, time, row, collector, timeDelta) -> { + throw target.invalidDataType(); + }; + }; + + static final TypeService NON_NEGATIVE_DERIVATIVE_OPERATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (target, time, row, collector, timeDelta) -> { + int current = row.getInt(0); + collector.putDouble(time, Math.abs(current - target.previousInt) / timeDelta); + target.previousInt = current; + }; + case INT64 -> + (target, time, row, collector, timeDelta) -> { + long current = row.getLong(0); + collector.putDouble(time, Math.abs(current - target.previousLong) / timeDelta); + target.previousLong = current; + }; + case FLOAT -> + (target, time, row, collector, timeDelta) -> { + float current = row.getFloat(0); + collector.putDouble(time, Math.abs(current - target.previousFloat) / timeDelta); + target.previousFloat = current; + }; + case DOUBLE -> + (target, time, row, collector, timeDelta) -> { + double current = row.getDouble(0); + collector.putDouble(time, Math.abs(current - target.previousDouble) / timeDelta); + target.previousDouble = current; + }; + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (target, time, row, collector, timeDelta) -> { + throw target.invalidDataType(); + }; + }; + + // UDF Row has no generic numeric accessor, so bind its primitive getter once during beforeStart. + static final TypeService NUMERIC_ROW_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> row -> row.getInt(0); + case INT64 -> row -> row.getLong(0); + case FLOAT -> row -> row.getFloat(0); + case DOUBLE -> row -> row.getDouble(0); + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + row -> { + throw invalidNumericDataType(type); + }; + }; + + // TsFile Type provides primitive numeric conversion for every supported column implementation. + static final TypeService NUMERIC_COLUMN_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, INT64, FLOAT, DOUBLE -> type::getDouble; + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (column, position) -> { + throw invalidNumericDataType(type); + }; + }; + + static final TypeService NUMERIC_ROW_COLLECTOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> (row, collector) -> collector.putInt(row.getTime(), row.getInt(0)); + case INT64 -> (row, collector) -> collector.putLong(row.getTime(), row.getLong(0)); + case FLOAT -> (row, collector) -> collector.putFloat(row.getTime(), row.getFloat(0)); + case DOUBLE -> (row, collector) -> collector.putDouble(row.getTime(), row.getDouble(0)); + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (row, collector) -> { + throw invalidNumericDataType(type); + }; + }; + + static final TypeService ABS_ROW_COLLECTOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (row, collector) -> collector.putInt(row.getTime(), Math.abs(row.getInt(0))); + case INT64 -> + (row, collector) -> collector.putLong(row.getTime(), Math.abs(row.getLong(0))); + case FLOAT -> + (row, collector) -> collector.putFloat(row.getTime(), Math.abs(row.getFloat(0))); + case DOUBLE -> + (row, collector) -> collector.putDouble(row.getTime(), Math.abs(row.getDouble(0))); + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (row, collector) -> { + throw invalidNumericDataType(type); + }; + }; + + static final TypeService ABS_ROW_MAPPER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> row -> Math.abs(row.getInt(0)); + case INT64 -> row -> Math.abs(row.getLong(0)); + case FLOAT -> row -> Math.abs(row.getFloat(0)); + case DOUBLE -> row -> Math.abs(row.getDouble(0)); + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + row -> { + throw invalidNumericDataType(type); + }; + }; + + static final TypeService ABS_COLUMN_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> UDTFAbs::transformInt; + case INT64 -> UDTFAbs::transformLong; + case FLOAT -> UDTFAbs::transformFloat; + case DOUBLE -> UDTFAbs::transformDouble; + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (target, columns, builder) -> { + throw invalidNumericDataType(type); + }; + }; + + static final TypeService> M4_WINDOW_TRANSFORMER_SERVICE = + numericWindowTransformerService( + UDTFM4::transformInt, + UDTFM4::transformLong, + UDTFM4::transformFloat, + UDTFM4::transformDouble); + + static final TypeService> + BUCKET_M4_WINDOW_TRANSFORMER_SERVICE = + numericWindowTransformerService( + UDTFEqualSizeBucketM4Sample::transformInt, + UDTFEqualSizeBucketM4Sample::transformLong, + UDTFEqualSizeBucketM4Sample::transformFloat, + UDTFEqualSizeBucketM4Sample::transformDouble); + + static final TypeService> + BUCKET_AGG_WINDOW_TRANSFORMER_SERVICE = + numericWindowTransformerService( + UDTFEqualSizeBucketAggSample::aggregateInt, + UDTFEqualSizeBucketAggSample::aggregateLong, + UDTFEqualSizeBucketAggSample::aggregateFloat, + UDTFEqualSizeBucketAggSample::aggregateDouble); + + static final TypeService> + BUCKET_OUTLIER_WINDOW_TRANSFORMER_SERVICE = + numericWindowTransformerService( + UDTFEqualSizeBucketOutlierSample::outlierSampleInt, + UDTFEqualSizeBucketOutlierSample::outlierSampleLong, + UDTFEqualSizeBucketOutlierSample::outlierSampleFloat, + UDTFEqualSizeBucketOutlierSample::outlierSampleDouble); + + static final TypeService CHANGE_POINT_PROCESSOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> UDTFChangePoints::transformBoolean; + case INT32 -> UDTFChangePoints::transformInt; + case INT64 -> UDTFChangePoints::transformLong; + case FLOAT -> UDTFChangePoints::transformFloat; + case DOUBLE -> UDTFChangePoints::transformDouble; + case TEXT -> UDTFChangePoints::transformString; + case DATE, TIMESTAMP, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (target, row, collector) -> {}; + }; + + static final TypeService SELECT_K_ROW_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> (target, row) -> target.transformInt(row.getTime(), row.getInt(0)); + case INT64, TIMESTAMP -> + (target, row) -> target.transformLong(row.getTime(), row.getLong(0)); + case FLOAT -> (target, row) -> target.transformFloat(row.getTime(), row.getFloat(0)); + case DOUBLE -> (target, row) -> target.transformDouble(row.getTime(), row.getDouble(0)); + case TEXT, STRING -> + (target, row) -> target.transformString(row.getTime(), row.getString(0)); + case BOOLEAN, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (target, row) -> { + throw invalidSelectKDataType(type); + }; + }; + + static final TypeService SELECT_K_TERMINATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> UDTFSelectK::terminateInt; + case INT64, TIMESTAMP -> UDTFSelectK::terminateLong; + case FLOAT -> UDTFSelectK::terminateFloat; + case DOUBLE -> UDTFSelectK::terminateDouble; + case TEXT, STRING -> UDTFSelectK::terminateString; + case BOOLEAN, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (target, collector) -> { + throw invalidSelectKDataType(type); + }; + }; + + static final TypeService TOP_K_QUEUE_CONSTRUCTOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> UDTFTopK::initializeIntQueue; + case INT64, TIMESTAMP -> UDTFTopK::initializeLongQueue; + case FLOAT -> UDTFTopK::initializeFloatQueue; + case DOUBLE -> UDTFTopK::initializeDoubleQueue; + case TEXT, STRING -> UDTFTopK::initializeStringQueue; + case BOOLEAN, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + target -> { + throw invalidSelectKDataType(type); + }; + }; + + static final TypeService BOTTOM_K_QUEUE_CONSTRUCTOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> UDTFBottomK::initializeIntQueue; + case INT64, TIMESTAMP -> UDTFBottomK::initializeLongQueue; + case FLOAT -> UDTFBottomK::initializeFloatQueue; + case DOUBLE -> UDTFBottomK::initializeDoubleQueue; + case TEXT, STRING -> UDTFBottomK::initializeStringQueue; + case BOOLEAN, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + target -> { + throw invalidSelectKDataType(type); + }; + }; + + static final TypeService CONSTANT_PARSER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> UDTFConst::parseInt; + case DATE -> UDTFConst::parseDate; + case INT64, TIMESTAMP -> UDTFConst::parseLong; + case FLOAT -> UDTFConst::parseFloat; + case DOUBLE -> UDTFConst::parseDouble; + case BOOLEAN -> UDTFConst::parseBoolean; + case TEXT, STRING -> UDTFConst::parseText; + case BLOB, OBJECT -> UDTFConst::parseBlob; + case ROW, UNKNOWN, VECTOR -> + (target, parameters) -> { + throw new UnsupportedOperationException(); + }; + }; + + static final TypeService CONSTANT_ROW_COLLECTOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (target, row, collector) -> collector.putInt(row.getTime(), target.intValue()); + case INT64, TIMESTAMP -> + (target, row, collector) -> collector.putLong(row.getTime(), target.longValue()); + case FLOAT -> + (target, row, collector) -> collector.putFloat(row.getTime(), target.floatValue()); + case DOUBLE -> + (target, row, collector) -> + collector.putDouble(row.getTime(), target.doubleValue()); + case BOOLEAN -> + (target, row, collector) -> + collector.putBoolean(row.getTime(), target.booleanValue()); + case TEXT, STRING, BLOB, OBJECT -> + (target, row, collector) -> + collector.putBinary(row.getTime(), target.binaryValue()); + case ROW, UNKNOWN, VECTOR -> + (target, row, collector) -> { + throw new UnsupportedOperationException(); + }; + }; + + static final TypeService CONSTANT_ROW_MAPPER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> UDTFConst::intValue; + case INT64, TIMESTAMP -> UDTFConst::longValue; + case FLOAT -> UDTFConst::floatValue; + case DOUBLE -> UDTFConst::doubleValue; + case BOOLEAN -> UDTFConst::booleanValue; + case TEXT, STRING, BLOB, OBJECT -> UDTFConst::binaryValue; + case ROW, UNKNOWN, VECTOR -> + target -> { + throw new UnsupportedOperationException(); + }; + }; + + static final TypeService CONSTANT_COLUMN_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> (target, builder) -> builder.writeInt(target.intValue()); + case INT64, TIMESTAMP -> (target, builder) -> builder.writeLong(target.longValue()); + case FLOAT -> (target, builder) -> builder.writeFloat(target.floatValue()); + case DOUBLE -> (target, builder) -> builder.writeDouble(target.doubleValue()); + case BOOLEAN -> (target, builder) -> builder.writeBoolean(target.booleanValue()); + case TEXT, STRING, BLOB, OBJECT -> + (target, builder) -> builder.writeBinary(target.tsFileBinaryValue()); + case ROW, UNKNOWN, VECTOR -> + (target, builder) -> { + throw new UnsupportedOperationException(); + }; + }; + + static final TypeService + CONTINUOUSLY_SATISFY_ROW_TRANSFORMER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> (target, row) -> target.transformInt(row.getTime(), row.getInt(0)); + case INT64 -> (target, row) -> target.transformLong(row.getTime(), row.getLong(0)); + case FLOAT -> + (target, row) -> target.transformFloat(row.getTime(), row.getFloat(0)); + case DOUBLE -> + (target, row) -> target.transformDouble(row.getTime(), row.getDouble(0)); + case BOOLEAN -> + (target, row) -> target.transformBoolean(row.getTime(), row.getBoolean(0)); + case DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (target, row) -> { + throw invalidContinuouslySatisfyDataType(type); + }; + }; + + static final TypeService CONTINUOUSLY_SATISFY_TERMINATOR_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, INT64, FLOAT, DOUBLE, BOOLEAN -> + UDTFContinuouslySatisfy::terminateSupportedType; + case DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (target, collector) -> { + throw invalidContinuouslySatisfyDataType(type); + }; + }; + + static { + VALUE_TREND_READER_SERVICE.check(); + VALUE_DIFFERENCE_OPERATOR_SERVICE.check(); + NON_NEGATIVE_VALUE_DIFFERENCE_OPERATOR_SERVICE.check(); + DERIVATIVE_OPERATOR_SERVICE.check(); + NON_NEGATIVE_DERIVATIVE_OPERATOR_SERVICE.check(); + NUMERIC_ROW_READER_SERVICE.check(); + NUMERIC_COLUMN_READER_SERVICE.check(); + NUMERIC_ROW_COLLECTOR_SERVICE.check(); + ABS_ROW_COLLECTOR_SERVICE.check(); + ABS_ROW_MAPPER_SERVICE.check(); + ABS_COLUMN_TRANSFORMER_SERVICE.check(); + M4_WINDOW_TRANSFORMER_SERVICE.check(); + BUCKET_M4_WINDOW_TRANSFORMER_SERVICE.check(); + BUCKET_AGG_WINDOW_TRANSFORMER_SERVICE.check(); + BUCKET_OUTLIER_WINDOW_TRANSFORMER_SERVICE.check(); + CHANGE_POINT_PROCESSOR_SERVICE.check(); + SELECT_K_ROW_TRANSFORMER_SERVICE.check(); + SELECT_K_TERMINATOR_SERVICE.check(); + TOP_K_QUEUE_CONSTRUCTOR_SERVICE.check(); + BOTTOM_K_QUEUE_CONSTRUCTOR_SERVICE.check(); + CONSTANT_PARSER_SERVICE.check(); + CONSTANT_ROW_COLLECTOR_SERVICE.check(); + CONSTANT_ROW_MAPPER_SERVICE.check(); + CONSTANT_COLUMN_VALUE_WRITER_SERVICE.check(); + CONTINUOUSLY_SATISFY_ROW_TRANSFORMER_SERVICE.check(); + CONTINUOUSLY_SATISFY_TERMINATOR_SERVICE.check(); + } + + private TypeServices() {} + + private static UDFInputSeriesDataTypeNotValidException invalidNumericDataType( + org.apache.tsfile.read.common.type.Type type) { + return new UDFInputSeriesDataTypeNotValidException( + 0, + UDFDataTypeTransformer.transformReadTypeToUDFDataType(type), + Type.INT32, + Type.INT64, + Type.FLOAT, + Type.DOUBLE); + } + + private static UDFInputSeriesDataTypeNotValidException invalidSelectKDataType( + org.apache.tsfile.read.common.type.Type type) { + return new UDFInputSeriesDataTypeNotValidException( + 0, + UDFDataTypeTransformer.transformReadTypeToUDFDataType(type), + Type.INT32, + Type.INT64, + Type.FLOAT, + Type.DOUBLE, + Type.TEXT, + Type.DATE, + Type.TIMESTAMP, + Type.STRING); + } + + private static UDFInputSeriesDataTypeNotValidException invalidContinuouslySatisfyDataType( + org.apache.tsfile.read.common.type.Type type) { + return new UDFInputSeriesDataTypeNotValidException( + 0, + UDFDataTypeTransformer.transformReadTypeToUDFDataType(type), + Type.INT32, + Type.INT64, + Type.FLOAT, + Type.DOUBLE); + } + + // Keep each window algorithm's primitive output type while selecting its callback once. + private static TypeService> numericWindowTransformerService( + NumericWindowTransformer intTransformer, + NumericWindowTransformer longTransformer, + NumericWindowTransformer floatTransformer, + NumericWindowTransformer doubleTransformer) { + return type -> + switch (type.getTypeEnum()) { + case INT32 -> intTransformer; + case INT64 -> longTransformer; + case FLOAT -> floatTransformer; + case DOUBLE -> doubleTransformer; + case BOOLEAN, DATE, TIMESTAMP, TEXT, STRING, BLOB, OBJECT, ROW, UNKNOWN, VECTOR -> + (target, rowWindow, collector) -> { + throw invalidNumericDataType(type); + }; + }; + } + + @FunctionalInterface + interface PreviousValueReader { + void read(UDTFValueTrend target, Row row) + throws UDFInputSeriesDataTypeNotValidException, IOException; + } + + @FunctionalInterface + interface ValueDifferenceOperator { + void apply(UDTFValueTrend target, long time, Row row, PointCollector collector) + throws UDFInputSeriesDataTypeNotValidException, IOException; + } + + @FunctionalInterface + interface DerivativeOperator { + void apply( + UDTFValueTrend target, long time, Row row, PointCollector collector, double timeDelta) + throws UDFInputSeriesDataTypeNotValidException, IOException; + } + + @FunctionalInterface + interface NumericRowReader { + double read(Row row) throws UDFInputSeriesDataTypeNotValidException, IOException; + } + + @FunctionalInterface + interface NumericColumnReader { + double read(Column column, int position) throws UDFInputSeriesDataTypeNotValidException; + } + + @FunctionalInterface + interface NumericRowCollector { + void collect(Row row, PointCollector collector) + throws UDFInputSeriesDataTypeNotValidException, IOException; + } + + @FunctionalInterface + interface AbsRowCollector { + void collect(Row row, PointCollector collector) + throws UDFInputSeriesDataTypeNotValidException, IOException; + } + + @FunctionalInterface + interface AbsRowMapper { + Object map(Row row) throws UDFInputSeriesDataTypeNotValidException, IOException; + } + + @FunctionalInterface + interface AbsColumnTransformer { + void transform(UDTFAbs target, Column[] columns, ColumnBuilder builder) + throws UDFInputSeriesDataTypeNotValidException; + } + + @FunctionalInterface + interface NumericWindowTransformer { + void transform(T target, RowWindow rowWindow, PointCollector collector) + throws UDFInputSeriesDataTypeNotValidException, IOException; + } + + @FunctionalInterface + interface ChangePointProcessor { + void transform(UDTFChangePoints target, Row row, PointCollector collector) throws IOException; + } + + @FunctionalInterface + interface SelectKRowTransformer { + void transform(UDTFSelectK target, Row row) + throws UDFInputSeriesDataTypeNotValidException, IOException; + } + + @FunctionalInterface + interface SelectKTerminator { + void terminate(UDTFSelectK target, PointCollector collector) + throws UDFInputSeriesDataTypeNotValidException, IOException; + } + + @FunctionalInterface + interface TopKQueueConstructor { + void construct(UDTFTopK target); + } + + @FunctionalInterface + interface BottomKQueueConstructor { + void construct(UDTFBottomK target); + } + + @FunctionalInterface + interface ConstantParser { + void parse(UDTFConst target, UDFParameters parameters); + } + + @FunctionalInterface + interface ConstantRowCollector { + void collect(UDTFConst target, Row row, PointCollector collector) throws IOException; + } + + @FunctionalInterface + interface ConstantRowMapper { + Object map(UDTFConst target); + } + + @FunctionalInterface + interface ConstantColumnValueWriter { + void write(UDTFConst target, ColumnBuilder builder); + } + + @FunctionalInterface + interface ContinuouslySatisfyRowTransformer { + boolean transform(UDTFContinuouslySatisfy target, Row row) + throws UDFInputSeriesDataTypeNotValidException, IOException; + } + + @FunctionalInterface + interface ContinuouslySatisfyTerminator { + void terminate(UDTFContinuouslySatisfy target, PointCollector collector) + throws UDFInputSeriesDataTypeNotValidException, IOException; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java index 2af7ee3c83d5..06303b8d2a30 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java @@ -27,62 +27,36 @@ import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.MappableRowByRowAccessStrategy; -import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; -import org.apache.iotdb.udf.api.type.Type; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import java.io.IOException; @SuppressWarnings("java:S2177") public class UDTFAbs extends UDTFMath { + private TypeServices.AbsRowCollector rowCollector; + private TypeServices.AbsRowMapper rowMapper; + private TypeServices.AbsColumnTransformer columnTransformer; + @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws MetadataException { dataType = UDFDataTypeTransformer.transformToTsDataType(parameters.getDataType(0)); + Type type = Type.fromTsDataType(dataType); + rowCollector = TypeServices.ABS_ROW_COLLECTOR_SERVICE.call(type); + rowMapper = TypeServices.ABS_ROW_MAPPER_SERVICE.call(type); + columnTransformer = TypeServices.ABS_COLUMN_TRANSFORMER_SERVICE.call(type); configurations .setAccessStrategy(new MappableRowByRowAccessStrategy()) .setOutputDataType(UDFDataTypeTransformer.transformToUDFDataType(dataType)); } @Override - public void transform(Row row, PointCollector collector) - throws UDFInputSeriesDataTypeNotValidException, IOException { - long time = row.getTime(); - switch (dataType) { - case INT32: - collector.putInt(time, Math.abs(row.getInt(0))); - break; - case INT64: - collector.putLong(time, Math.abs(row.getLong(0))); - break; - case FLOAT: - collector.putFloat(time, Math.abs(row.getFloat(0))); - break; - case DOUBLE: - collector.putDouble(time, Math.abs(row.getDouble(0))); - break; - case BLOB: - case OBJECT: - case STRING: - case TIMESTAMP: - case TEXT: - case DATE: - case BOOLEAN: - case VECTOR: - case UNKNOWN: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + public void transform(Row row, PointCollector collector) throws IOException { + rowCollector.collect(row, collector); } @Override @@ -90,66 +64,12 @@ public Object transform(Row row) throws IOException { if (row.isNull(0)) { return null; } - switch (dataType) { - case INT32: - return Math.abs(row.getInt(0)); - case INT64: - return Math.abs(row.getLong(0)); - case FLOAT: - return Math.abs(row.getFloat(0)); - case DOUBLE: - return Math.abs(row.getDouble(0)); - case DATE: - case BOOLEAN: - case TEXT: - case TIMESTAMP: - case STRING: - case BLOB: - case OBJECT: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + return rowMapper.map(row); } @Override public void transform(Column[] columns, ColumnBuilder builder) throws Exception { - switch (dataType) { - case INT32: - transformInt(columns, builder); - return; - case INT64: - transformLong(columns, builder); - return; - case FLOAT: - transformFloat(columns, builder); - return; - case DOUBLE: - transformDouble(columns, builder); - return; - case BLOB: - case OBJECT: - case STRING: - case TEXT: - case TIMESTAMP: - case BOOLEAN: - case DATE: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + columnTransformer.transform(this, columns, builder); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java index 045e49b064b7..b4702c8046b2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java @@ -19,10 +19,6 @@ package org.apache.iotdb.commons.udf.builtin; -import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; -import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; -import org.apache.iotdb.udf.api.type.Type; - import org.apache.tsfile.utils.Pair; import java.util.Comparator; @@ -32,63 +28,50 @@ public class UDTFBottomK extends UDTFSelectK { @Override - protected void constructPQ() throws UDFInputSeriesDataTypeNotValidException { - switch (dataType) { - case INT32: - case DATE: - intPQ = new PriorityQueue<>(k, Comparator.comparing(o -> -o.right)); - break; - case INT64: - case TIMESTAMP: - longPQ = new PriorityQueue<>(k, Comparator.comparing(o -> -o.right)); - break; - case FLOAT: - floatPQ = new PriorityQueue<>(k, Comparator.comparing(o -> -o.right)); - break; - case DOUBLE: - doublePQ = new PriorityQueue<>(k, Comparator.comparing(o -> -o.right)); - break; - case TEXT: - case STRING: - stringPQ = - new PriorityQueue<>( - k, - (pairA, pairB) -> { - final String cs1 = pairB.right; - final String cs2 = pairA.right; - - if (Objects.requireNonNull(cs1).equals(Objects.requireNonNull(cs2))) { - return 0; - } - - for (int i = 0, len = Math.min(cs1.length(), cs2.length()); i < len; i++) { - final char a = cs1.charAt(i); - final char b = cs2.charAt(i); - if (a != b) { - return a - b; - } - } - - return cs1.length() - cs2.length(); - }); - break; - case BLOB: - case OBJECT: - case BOOLEAN: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE, - Type.TEXT, - Type.DATE, - Type.TIMESTAMP, - Type.STRING); - } + protected void constructPQ() { + TypeServices.BOTTOM_K_QUEUE_CONSTRUCTOR_SERVICE + .call(org.apache.tsfile.read.common.type.Type.fromTsDataType(dataType)) + .construct(this); + } + + void initializeIntQueue() { + intPQ = new PriorityQueue<>(k, Comparator.comparing(o -> -o.right)); + } + + void initializeLongQueue() { + longPQ = new PriorityQueue<>(k, Comparator.comparing(o -> -o.right)); + } + + void initializeFloatQueue() { + floatPQ = new PriorityQueue<>(k, Comparator.comparing(o -> -o.right)); + } + + void initializeDoubleQueue() { + doublePQ = new PriorityQueue<>(k, Comparator.comparing(o -> -o.right)); + } + + void initializeStringQueue() { + stringPQ = + new PriorityQueue<>( + k, + (pairA, pairB) -> { + final String cs1 = pairB.right; + final String cs2 = pairA.right; + + if (Objects.requireNonNull(cs1).equals(Objects.requireNonNull(cs2))) { + return 0; + } + + for (int i = 0, len = Math.min(cs1.length(), cs2.length()); i < len; i++) { + final char a = cs1.charAt(i); + final char b = cs2.charAt(i); + if (a != b) { + return a - b; + } + } + + return cs1.length() - cs2.length(); + }); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFChangePoints.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFChangePoints.java index dd19304384b8..f573e0e320ba 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFChangePoints.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFChangePoints.java @@ -19,6 +19,7 @@ package org.apache.iotdb.commons.udf.builtin; +import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; @@ -28,19 +29,21 @@ import org.apache.iotdb.udf.api.customizer.strategy.RowByRowAccessStrategy; import org.apache.iotdb.udf.api.type.Type; +import java.io.IOException; + /** * Return a series that the consecutive identical values in input series are removed (keeping only * the first one). */ public class UDTFChangePoints implements UDTF { private boolean isFirst = true; - private Type dataType; private boolean cacheBoolean; private int cacheInt; private long cacheLong; private float cacheFloat; private double cacheDouble; private String cacheString; + private TypeServices.ChangePointProcessor changePointProcessor; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -50,96 +53,99 @@ public void validate(UDFParameterValidator validator) throws Exception { @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { - dataType = parameters.getDataType(0); + Type dataType = parameters.getDataType(0); + changePointProcessor = + TypeServices.CHANGE_POINT_PROCESSOR_SERVICE.call( + UDFDataTypeTransformer.transformUDFDataTypeToReadType(dataType)); configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(dataType); } @Override public void transform(Row row, PointCollector collector) throws Exception { - switch (dataType) { - case BOOLEAN: - if (isFirst) { - isFirst = false; - cacheBoolean = row.getBoolean(0); - collector.putBoolean(row.getTime(), cacheBoolean); - } else { - boolean rowData = row.getBoolean(0); - if (rowData != cacheBoolean) { - cacheBoolean = rowData; - collector.putBoolean(row.getTime(), cacheBoolean); - } - } - break; - case INT32: - if (isFirst) { - isFirst = false; - cacheInt = row.getInt(0); - collector.putInt(row.getTime(), cacheInt); - } else { - int rowData = row.getInt(0); - if (rowData != cacheInt) { - cacheInt = rowData; - collector.putInt(row.getTime(), cacheInt); - } - } - break; - case INT64: - if (isFirst) { - isFirst = false; - cacheLong = row.getLong(0); - collector.putLong(row.getTime(), cacheLong); - } else { - long rowData = row.getLong(0); - if (rowData != cacheLong) { - cacheLong = rowData; - collector.putLong(row.getTime(), cacheLong); - } - } - break; - case FLOAT: - if (isFirst) { - isFirst = false; - cacheFloat = row.getFloat(0); - collector.putFloat(row.getTime(), cacheFloat); - } else { - float rowData = row.getFloat(0); - if (rowData != cacheFloat) { - cacheFloat = rowData; - collector.putFloat(row.getTime(), cacheFloat); - } - } - break; - case DOUBLE: - if (isFirst) { - isFirst = false; - cacheDouble = row.getDouble(0); - collector.putDouble(row.getTime(), cacheDouble); - } else { - double rowData = row.getDouble(0); - if (rowData != cacheDouble) { - cacheDouble = rowData; - collector.putDouble(row.getTime(), cacheDouble); - } - } - break; - case TEXT: - if (isFirst) { - isFirst = false; - cacheString = row.getString(0); - collector.putString(row.getTime(), cacheString); - } else { - String rowData = row.getString(0); - if (!rowData.equals(cacheString)) { - cacheString = rowData; - collector.putString(row.getTime(), cacheString); - } - } - case STRING: - case BLOB: - case DATE: - case TIMESTAMP: - default: - break; + changePointProcessor.transform(this, row, collector); + } + + void transformBoolean(Row row, PointCollector collector) throws IOException { + if (isFirst) { + isFirst = false; + cacheBoolean = row.getBoolean(0); + collector.putBoolean(row.getTime(), cacheBoolean); + } else { + boolean rowData = row.getBoolean(0); + if (rowData != cacheBoolean) { + cacheBoolean = rowData; + collector.putBoolean(row.getTime(), cacheBoolean); + } + } + } + + void transformInt(Row row, PointCollector collector) throws IOException { + if (isFirst) { + isFirst = false; + cacheInt = row.getInt(0); + collector.putInt(row.getTime(), cacheInt); + } else { + int rowData = row.getInt(0); + if (rowData != cacheInt) { + cacheInt = rowData; + collector.putInt(row.getTime(), cacheInt); + } + } + } + + void transformLong(Row row, PointCollector collector) throws IOException { + if (isFirst) { + isFirst = false; + cacheLong = row.getLong(0); + collector.putLong(row.getTime(), cacheLong); + } else { + long rowData = row.getLong(0); + if (rowData != cacheLong) { + cacheLong = rowData; + collector.putLong(row.getTime(), cacheLong); + } + } + } + + void transformFloat(Row row, PointCollector collector) throws IOException { + if (isFirst) { + isFirst = false; + cacheFloat = row.getFloat(0); + collector.putFloat(row.getTime(), cacheFloat); + } else { + float rowData = row.getFloat(0); + if (rowData != cacheFloat) { + cacheFloat = rowData; + collector.putFloat(row.getTime(), cacheFloat); + } + } + } + + void transformDouble(Row row, PointCollector collector) throws IOException { + if (isFirst) { + isFirst = false; + cacheDouble = row.getDouble(0); + collector.putDouble(row.getTime(), cacheDouble); + } else { + double rowData = row.getDouble(0); + if (rowData != cacheDouble) { + cacheDouble = rowData; + collector.putDouble(row.getTime(), cacheDouble); + } + } + } + + void transformString(Row row, PointCollector collector) throws IOException { + if (isFirst) { + isFirst = false; + cacheString = row.getString(0); + collector.putString(row.getTime(), cacheString); + } else { + String rowData = row.getString(0); + if (!rowData.equals(cacheString)) { + cacheString = rowData; + collector.putString(row.getTime(), cacheString); + } } } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java index cb5cbcc0486b..0a80ad2de287 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java @@ -19,11 +19,9 @@ package org.apache.iotdb.commons.udf.builtin; -import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; -import org.apache.iotdb.udf.api.type.Type; import java.io.IOException; @@ -34,44 +32,7 @@ protected void doTransform(Row row, PointCollector collector) throws UDFInputSeriesDataTypeNotValidException, IOException { long currentTime = row.getTime(); double timeDelta = (double) currentTime - previousTime; - switch (dataType) { - case INT32: - int currentInt = row.getInt(0); - collector.putDouble(currentTime, (currentInt - previousInt) / timeDelta); - previousInt = currentInt; - break; - case INT64: - long currentLong = row.getLong(0); - collector.putDouble(currentTime, (currentLong - previousLong) / timeDelta); - previousLong = currentLong; - break; - case FLOAT: - float currentFloat = row.getFloat(0); - collector.putDouble(currentTime, (currentFloat - previousFloat) / timeDelta); - previousFloat = currentFloat; - break; - case DOUBLE: - double currentDouble = row.getDouble(0); - collector.putDouble(currentTime, (currentDouble - previousDouble) / timeDelta); - previousDouble = currentDouble; - break; - case DATE: - case BOOLEAN: - case TIMESTAMP: - case TEXT: - case STRING: - case BLOB: - case OBJECT: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + derivativeOperator.apply(this, currentTime, row, collector, timeDelta); previousTime = currentTime; } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java index 7f8d81f6b829..0a4deac2024e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java @@ -19,11 +19,9 @@ package org.apache.iotdb.commons.udf.builtin; -import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; -import org.apache.iotdb.udf.api.type.Type; import java.io.IOException; @@ -32,44 +30,6 @@ public class UDTFCommonValueDifference extends UDTFValueDifference { @Override protected void doTransform(Row row, PointCollector collector) throws UDFInputSeriesDataTypeNotValidException, IOException { - long time = row.getTime(); - switch (dataType) { - case INT32: - int currentInt = row.getInt(0); - collector.putInt(time, currentInt - previousInt); - previousInt = currentInt; - break; - case INT64: - long currentLong = row.getLong(0); - collector.putLong(time, currentLong - previousLong); - previousLong = currentLong; - break; - case FLOAT: - float currentFloat = row.getFloat(0); - collector.putFloat(time, currentFloat - previousFloat); - previousFloat = currentFloat; - break; - case DOUBLE: - double currentDouble = row.getDouble(0); - collector.putDouble(time, currentDouble - previousDouble); - previousDouble = currentDouble; - break; - case STRING: - case BLOB: - case OBJECT: - case TIMESTAMP: - case TEXT: - case BOOLEAN: - case DATE: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + valueDifferenceOperator.apply(this, row.getTime(), row, collector); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java index f3436859f2a4..09ebb443a8d6 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java @@ -34,6 +34,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.DateUtils; @@ -60,14 +61,15 @@ public class UDTFConst implements UDTF { VALID_TYPES.add(TSDataType.OBJECT.name()); } - private TSDataType dataType; - private int intValue; private long longValue; private float floatValue; private double doubleValue; private boolean booleanValue; private Binary binaryValue; + private TypeServices.ConstantRowCollector rowCollector; + private TypeServices.ConstantRowMapper rowMapper; + private TypeServices.ConstantColumnValueWriter columnValueWriter; @Override public void validate(UDFParameterValidator validator) throws UDFParameterNotValidException { @@ -82,38 +84,12 @@ public void validate(UDFParameterValidator validator) throws UDFParameterNotVali @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) { - dataType = TSDataType.valueOf(parameters.getString("type")); - switch (dataType) { - case INT32: - intValue = Integer.parseInt(parameters.getString("value")); - break; - case DATE: - intValue = DateUtils.parseDateExpressionToInt(parameters.getString("value")); - break; - case INT64: - case TIMESTAMP: - longValue = Long.parseLong(parameters.getString("value")); - break; - case FLOAT: - floatValue = Float.parseFloat(parameters.getString("value")); - break; - case DOUBLE: - doubleValue = Double.parseDouble(parameters.getString("value")); - break; - case BOOLEAN: - booleanValue = Boolean.parseBoolean(parameters.getString("value")); - break; - case TEXT: - case STRING: - binaryValue = BytesUtils.valueOf(parameters.getString("value")); - break; - case BLOB: - case OBJECT: - binaryValue = new Binary(BlobUtils.parseBlobString(parameters.getString("value"))); - break; - default: - throw new UnsupportedOperationException(); - } + TSDataType dataType = TSDataType.valueOf(parameters.getString("type")); + Type type = Type.fromTsDataType(dataType); + TypeServices.CONSTANT_PARSER_SERVICE.call(type).parse(this, parameters); + rowCollector = TypeServices.CONSTANT_ROW_COLLECTOR_SERVICE.call(type); + rowMapper = TypeServices.CONSTANT_ROW_MAPPER_SERVICE.call(type); + columnValueWriter = TypeServices.CONSTANT_COLUMN_VALUE_WRITER_SERVICE.call(type); configurations .setAccessStrategy(new MappableRowByRowAccessStrategy()) @@ -122,162 +98,93 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { - switch (dataType) { - case INT32: - case DATE: - collector.putInt(row.getTime(), intValue); - break; - case INT64: - case TIMESTAMP: - collector.putLong(row.getTime(), longValue); - break; - case FLOAT: - collector.putFloat(row.getTime(), floatValue); - break; - case DOUBLE: - collector.putDouble(row.getTime(), doubleValue); - break; - case BOOLEAN: - collector.putBoolean(row.getTime(), booleanValue); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - collector.putBinary(row.getTime(), UDFBinaryTransformer.transformToUDFBinary(binaryValue)); - break; - default: - throw new UnsupportedOperationException(); - } + rowCollector.collect(this, row, collector); } @Override public Object transform(Row row) throws IOException { - switch (dataType) { - case INT32: - case DATE: - return intValue; - case INT64: - case TIMESTAMP: - return longValue; - case FLOAT: - return floatValue; - case DOUBLE: - return doubleValue; - case BOOLEAN: - return booleanValue; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - return UDFBinaryTransformer.transformToUDFBinary(binaryValue); - default: - throw new UnsupportedOperationException(); - } + return rowMapper.map(this); } @Override public void transform(Column[] columns, ColumnBuilder builder) throws Exception { - int count = columns[0].getPositionCount(); + writeConstant(columns, builder); + } - switch (dataType) { - case INT32: - case DATE: - for (int i = 0; i < count; i++) { - boolean hasWritten = false; - for (int j = 0; j < columns.length - 1; j++) { - if (!columns[j].isNull(i)) { - builder.writeInt(intValue); - hasWritten = true; - break; - } - } - if (!hasWritten) { - builder.appendNull(); - } - } - return; - case INT64: - case TIMESTAMP: - for (int i = 0; i < count; i++) { - boolean hasWritten = false; - for (int j = 0; j < columns.length - 1; j++) { - if (!columns[j].isNull(i)) { - builder.writeLong(longValue); - hasWritten = true; - break; - } - } - if (!hasWritten) { - builder.appendNull(); - } - } - return; - case FLOAT: - for (int i = 0; i < count; i++) { - boolean hasWritten = false; - for (int j = 0; j < columns.length - 1; j++) { - if (!columns[j].isNull(i)) { - builder.writeFloat(floatValue); - hasWritten = true; - break; - } - } - if (!hasWritten) { - builder.appendNull(); - } - } - return; - case DOUBLE: - for (int i = 0; i < count; i++) { - boolean hasWritten = false; - for (int j = 0; j < columns.length - 1; j++) { - if (!columns[j].isNull(i)) { - builder.writeDouble(doubleValue); - hasWritten = true; - break; - } - } - if (!hasWritten) { - builder.appendNull(); - } - } - return; - case BOOLEAN: - for (int i = 0; i < count; i++) { - boolean hasWritten = false; - for (int j = 0; j < columns.length - 1; j++) { - if (!columns[j].isNull(i)) { - builder.writeBoolean(booleanValue); - hasWritten = true; - break; - } - } - if (!hasWritten) { - builder.appendNull(); - } - } - return; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - for (int i = 0; i < count; i++) { - boolean hasWritten = false; - for (int j = 0; j < columns.length - 1; j++) { - if (!columns[j].isNull(i)) { - builder.writeBinary(binaryValue); - hasWritten = true; - break; - } - } - if (!hasWritten) { - builder.appendNull(); - } + void parseInt(UDFParameters parameters) { + intValue = Integer.parseInt(parameters.getString("value")); + } + + void parseDate(UDFParameters parameters) { + intValue = DateUtils.parseDateExpressionToInt(parameters.getString("value")); + } + + void parseLong(UDFParameters parameters) { + longValue = Long.parseLong(parameters.getString("value")); + } + + void parseFloat(UDFParameters parameters) { + floatValue = Float.parseFloat(parameters.getString("value")); + } + + void parseDouble(UDFParameters parameters) { + doubleValue = Double.parseDouble(parameters.getString("value")); + } + + void parseBoolean(UDFParameters parameters) { + booleanValue = Boolean.parseBoolean(parameters.getString("value")); + } + + void parseText(UDFParameters parameters) { + binaryValue = BytesUtils.valueOf(parameters.getString("value")); + } + + void parseBlob(UDFParameters parameters) { + binaryValue = new Binary(BlobUtils.parseBlobString(parameters.getString("value"))); + } + + int intValue() { + return intValue; + } + + long longValue() { + return longValue; + } + + float floatValue() { + return floatValue; + } + + double doubleValue() { + return doubleValue; + } + + boolean booleanValue() { + return booleanValue; + } + + org.apache.iotdb.udf.api.type.Binary binaryValue() { + return UDFBinaryTransformer.transformToUDFBinary(binaryValue); + } + + Binary tsFileBinaryValue() { + return binaryValue; + } + + private void writeConstant(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + for (int i = 0; i < count; i++) { + boolean hasWritten = false; + for (int j = 0; j < columns.length - 1; j++) { + if (!columns[j].isNull(i)) { + columnValueWriter.write(this, builder); + hasWritten = true; + break; } - return; - default: - throw new UnsupportedOperationException(); + } + if (!hasWritten) { + builder.appendNull(); + } } } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java index de6232ac6db3..657b75e60274 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java @@ -32,7 +32,6 @@ import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; import org.apache.iotdb.udf.api.type.Type; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Pair; import java.io.IOException; @@ -40,11 +39,12 @@ public abstract class UDTFContinuouslySatisfy implements UDTF { protected long min; protected long max; - protected TSDataType dataType; protected long satisfyValueCount; protected long satisfyValueLastTime; protected long satisfyValueStartTime; protected Pair interval; + private TypeServices.ContinuouslySatisfyRowTransformer rowTransformer; + private TypeServices.ContinuouslySatisfyTerminator terminator; @Override public void validate(UDFParameterValidator validator) throws UDFException { @@ -74,7 +74,12 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati satisfyValueStartTime = 0L; satisfyValueLastTime = -1L; - dataType = UDFDataTypeTransformer.transformToTsDataType(parameters.getDataType(0)); + rowTransformer = + TypeServices.CONTINUOUSLY_SATISFY_ROW_TRANSFORMER_SERVICE.call( + UDFDataTypeTransformer.transformUDFDataTypeToReadType(parameters.getDataType(0))); + terminator = + TypeServices.CONTINUOUSLY_SATISFY_TERMINATOR_SERVICE.call( + UDFDataTypeTransformer.transformUDFDataTypeToReadType(parameters.getDataType(0))); min = parameters.getLongOrDefault("min", getDefaultMin()); max = parameters.getLongOrDefault("max", getDefaultMax()); configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.INT64); @@ -83,40 +88,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws IOException, UDFInputSeriesDataTypeNotValidException { - boolean needAddNewRecord; - switch (dataType) { - case INT32: - needAddNewRecord = transformInt(row.getTime(), row.getInt(0)); - break; - case INT64: - needAddNewRecord = transformLong(row.getTime(), row.getLong(0)); - break; - case FLOAT: - needAddNewRecord = transformFloat(row.getTime(), row.getFloat(0)); - break; - case DOUBLE: - needAddNewRecord = transformDouble(row.getTime(), row.getDouble(0)); - break; - case BOOLEAN: - needAddNewRecord = transformBoolean(row.getTime(), row.getBoolean(0)); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - case TIMESTAMP: - case DATE: - default: - // This will not happen - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } - if (needAddNewRecord) { + if (rowTransformer.transform(this, row)) { collector.putLong(interval.left, interval.right); } } @@ -215,33 +187,12 @@ protected boolean transformBoolean(long time, boolean value) { @Override public void terminate(PointCollector collector) throws UDFInputSeriesDataTypeNotValidException, IOException { - switch (dataType) { - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - case BOOLEAN: - if (satisfyValueCount > 0) { - if (getRecord() >= min && getRecord() <= max) { - collector.putLong(satisfyValueStartTime, getRecord()); - } - } - break; - case TIMESTAMP: - case DATE: - case STRING: - case BLOB: - case OBJECT: - case TEXT: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); + terminator.terminate(this, collector); + } + + void terminateSupportedType(PointCollector collector) throws IOException { + if (satisfyValueCount > 0 && getRecord() >= min && getRecord() <= max) { + collector.putLong(satisfyValueStartTime, getRecord()); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFDerivative.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFDerivative.java index 81645d477878..2457ed898119 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFDerivative.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFDerivative.java @@ -29,16 +29,27 @@ import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; import org.apache.iotdb.udf.api.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; + import java.io.IOException; public abstract class UDTFDerivative extends UDTFValueTrend { protected long previousTime; + protected TypeServices.DerivativeOperator derivativeOperator; + + protected TypeService derivativeOperatorService() { + return TypeServices.DERIVATIVE_OPERATOR_SERVICE; + } @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws MetadataException { dataType = UDFDataTypeTransformer.transformToTsDataType(parameters.getDataType(0)); + org.apache.tsfile.read.common.type.Type type = + org.apache.tsfile.read.common.type.Type.fromTsDataType(dataType); + previousValueReader = TypeServices.VALUE_TREND_READER_SERVICE.call(type); + derivativeOperator = derivativeOperatorService().call(type); configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java index c16445b48b68..5f01cb625396 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java @@ -29,9 +29,7 @@ import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; import org.apache.iotdb.udf.api.exception.UDFException; -import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; import org.apache.iotdb.udf.api.exception.UDFParameterNotValidException; -import org.apache.iotdb.udf.api.type.Type; import org.apache.tsfile.enums.TSDataType; @@ -41,6 +39,7 @@ public class UDTFEqualSizeBucketAggSample extends UDTFEqualSizeBucketSample { private String aggMethodType; private Aggregator aggregator; + private TypeServices.NumericWindowTransformer windowTransformer; private interface Aggregator { @@ -465,40 +464,30 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati CommonMessages .EXCEPTION_ILLEGAL_AGGREGATION_METHOD_AGGREGATION_TYPE_SHOULD_AVG_MIN_MAX_SUM_2D7BEC96); } + windowTransformer = + TypeServices.BUCKET_AGG_WINDOW_TRANSFORMER_SERVICE.call( + UDFDataTypeTransformer.transformUDFDataTypeToReadType(parameters.getDataType(0))); } @Override public void transform(RowWindow rowWindow, PointCollector collector) throws IOException, UDFParameterNotValidException { - switch (dataType) { - case INT32: - aggregator.aggregateInt(rowWindow, collector); - break; - case INT64: - aggregator.aggregateLong(rowWindow, collector); - break; - case FLOAT: - aggregator.aggregateFloat(rowWindow, collector); - break; - case DOUBLE: - aggregator.aggregateDouble(rowWindow, collector); - break; - case BLOB: - case OBJECT: - case TEXT: - case DATE: - case STRING: - case TIMESTAMP: - case BOOLEAN: - default: - // This will not happen - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + windowTransformer.transform(this, rowWindow, collector); + } + + void aggregateInt(RowWindow rowWindow, PointCollector collector) throws IOException { + aggregator.aggregateInt(rowWindow, collector); + } + + void aggregateLong(RowWindow rowWindow, PointCollector collector) throws IOException { + aggregator.aggregateLong(rowWindow, collector); + } + + void aggregateFloat(RowWindow rowWindow, PointCollector collector) throws IOException { + aggregator.aggregateFloat(rowWindow, collector); + } + + void aggregateDouble(RowWindow rowWindow, PointCollector collector) throws IOException { + aggregator.aggregateDouble(rowWindow, collector); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java index fea5cb694fa7..b2e41eacbc78 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java @@ -27,16 +27,20 @@ import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; import org.apache.iotdb.udf.api.exception.UDFException; -import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; -import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.read.common.type.Type; import java.io.IOException; public class UDTFEqualSizeBucketM4Sample extends UDTFEqualSizeBucketSample { + private TypeServices.NumericWindowTransformer windowTransformer; + @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) { bucketSize *= 4; + windowTransformer = + TypeServices.BUCKET_M4_WINDOW_TRANSFORMER_SERVICE.call(Type.fromTsDataType(dataType)); configurations .setAccessStrategy(new SlidingSizeWindowAccessStrategy(bucketSize)) .setOutputDataType(UDFDataTypeTransformer.transformToUDFDataType(dataType)); @@ -45,36 +49,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(RowWindow rowWindow, PointCollector collector) throws UDFException, IOException { - switch (dataType) { - case INT32: - transformInt(rowWindow, collector); - break; - case INT64: - transformLong(rowWindow, collector); - break; - case FLOAT: - transformFloat(rowWindow, collector); - break; - case DOUBLE: - transformDouble(rowWindow, collector); - break; - case TIMESTAMP: - case BOOLEAN: - case DATE: - case STRING: - case TEXT: - case BLOB: - case OBJECT: - default: - // This will not happen - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + windowTransformer.transform(this, rowWindow, collector); } public void transformInt(RowWindow rowWindow, PointCollector collector) throws IOException { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java index e9b3fae53b1d..d28149954d78 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java @@ -30,9 +30,7 @@ import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; import org.apache.iotdb.udf.api.exception.UDFException; -import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; import org.apache.iotdb.udf.api.exception.UDFParameterNotValidException; -import org.apache.iotdb.udf.api.type.Type; import org.apache.tsfile.utils.Pair; @@ -48,6 +46,7 @@ public class UDTFEqualSizeBucketOutlierSample extends UDTFEqualSizeBucketSample private String type; private int number; private OutlierSampler outlierSampler; + private TypeServices.NumericWindowTransformer windowTransformer; private interface OutlierSampler { @@ -645,41 +644,31 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati CommonMessages .EXCEPTION_ILLEGAL_OUTLIER_METHOD_OUTLIER_TYPE_SHOULD_AVG_STENDIS_COS_PRENEXTDIS_91D1C70A); } + windowTransformer = + TypeServices.BUCKET_OUTLIER_WINDOW_TRANSFORMER_SERVICE.call( + UDFDataTypeTransformer.transformUDFDataTypeToReadType(parameters.getDataType(0))); } @Override public void transform(RowWindow rowWindow, PointCollector collector) throws IOException, UDFParameterNotValidException { - switch (dataType) { - case INT32: - outlierSampler.outlierSampleInt(rowWindow, collector); - break; - case INT64: - outlierSampler.outlierSampleLong(rowWindow, collector); - break; - case FLOAT: - outlierSampler.outlierSampleFloat(rowWindow, collector); - break; - case DOUBLE: - outlierSampler.outlierSampleDouble(rowWindow, collector); - break; - case TEXT: - case BLOB: - case OBJECT: - case DATE: - case STRING: - case BOOLEAN: - case TIMESTAMP: - default: - // This will not happen - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + windowTransformer.transform(this, rowWindow, collector); + } + + void outlierSampleInt(RowWindow rowWindow, PointCollector collector) throws IOException { + outlierSampler.outlierSampleInt(rowWindow, collector); + } + + void outlierSampleLong(RowWindow rowWindow, PointCollector collector) throws IOException { + outlierSampler.outlierSampleLong(rowWindow, collector); + } + + void outlierSampleFloat(RowWindow rowWindow, PointCollector collector) throws IOException { + outlierSampler.outlierSampleFloat(rowWindow, collector); + } + + void outlierSampleDouble(RowWindow rowWindow, PointCollector collector) throws IOException { + outlierSampler.outlierSampleDouble(rowWindow, collector); } public void addToMinHeap(PriorityQueue> pq, int i, double value) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java index 566e02e7b58f..7d2ad94de084 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java @@ -26,8 +26,8 @@ import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; -import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; -import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.read.common.type.Type; import java.io.IOException; import java.security.SecureRandom; @@ -35,48 +35,20 @@ public class UDTFEqualSizeBucketRandomSample extends UDTFEqualSizeBucketSample { private SecureRandom random; + private TypeServices.NumericRowCollector rowCollector; @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) { random = new SecureRandom(); + rowCollector = TypeServices.NUMERIC_ROW_COLLECTOR_SERVICE.call(Type.fromTsDataType(dataType)); configurations .setAccessStrategy(new SlidingSizeWindowAccessStrategy(bucketSize)) .setOutputDataType(UDFDataTypeTransformer.transformToUDFDataType(dataType)); } @Override - public void transform(RowWindow rowWindow, PointCollector collector) - throws IOException, UDFInputSeriesDataTypeNotValidException { + public void transform(RowWindow rowWindow, PointCollector collector) throws IOException { Row row = rowWindow.getRow(random.nextInt(rowWindow.windowSize())); - switch (dataType) { - case INT32: - collector.putInt(row.getTime(), row.getInt(0)); - break; - case INT64: - collector.putLong(row.getTime(), row.getLong(0)); - break; - case FLOAT: - collector.putFloat(row.getTime(), row.getFloat(0)); - break; - case DOUBLE: - collector.putDouble(row.getTime(), row.getDouble(0)); - break; - case BOOLEAN: - case TIMESTAMP: - case DATE: - case STRING: - case BLOB: - case OBJECT: - case TEXT: - default: - // This will not happen - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + rowCollector.collect(row, collector); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java index 4c17a2706973..4b3299f538c0 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java @@ -29,19 +29,18 @@ import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.MappableRowByRowAccessStrategy; import org.apache.iotdb.udf.api.exception.UDFException; -import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; import org.apache.iotdb.udf.api.type.Type; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.enums.TSDataType; import java.io.IOException; public class UDTFInRange implements UDTF { - protected TSDataType dataType; protected double upper; protected double lower; + private TypeServices.NumericRowReader rowReader; + private TypeServices.NumericColumnReader columnReader; @Override public void validate(UDFParameterValidator validator) throws UDFException { @@ -62,46 +61,20 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati throws MetadataException { upper = parameters.getDouble("upper"); lower = parameters.getDouble("lower"); - dataType = UDFDataTypeTransformer.transformToTsDataType(parameters.getDataType(0)); + org.apache.tsfile.read.common.type.Type type = + UDFDataTypeTransformer.transformUDFDataTypeToReadType(parameters.getDataType(0)); + rowReader = TypeServices.NUMERIC_ROW_READER_SERVICE.call(type); + columnReader = TypeServices.NUMERIC_COLUMN_READER_SERVICE.call(type); configurations .setAccessStrategy(new MappableRowByRowAccessStrategy()) .setOutputDataType(Type.BOOLEAN); } @Override - public void transform(Row row, PointCollector collector) - throws UDFInputSeriesDataTypeNotValidException, IOException { + public void transform(Row row, PointCollector collector) throws IOException { long time = row.getTime(); - switch (dataType) { - case INT32: - collector.putBoolean(time, row.getInt(0) >= lower && upper >= row.getInt(0)); - break; - case INT64: - collector.putBoolean(time, row.getLong(0) >= lower && upper >= row.getLong(0)); - break; - case FLOAT: - collector.putBoolean(time, row.getFloat(0) >= lower && upper >= row.getFloat(0)); - break; - case DOUBLE: - collector.putBoolean(time, row.getDouble(0) >= lower && upper >= row.getDouble(0)); - break; - case BLOB: - case OBJECT: - case TEXT: - case DATE: - case STRING: - case TIMESTAMP: - case BOOLEAN: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + double value = rowReader.read(row); + collector.putBoolean(time, value >= lower && upper >= value); } @Override @@ -109,124 +82,21 @@ public Object transform(Row row) throws Exception { if (row.isNull(0)) { return null; } - switch (dataType) { - case INT32: - return row.getInt(0) >= lower && upper >= row.getInt(0); - case INT64: - return row.getLong(0) >= lower && upper >= row.getLong(0); - case FLOAT: - return row.getFloat(0) >= lower && upper >= row.getFloat(0); - case DOUBLE: - return row.getDouble(0) >= lower && upper >= row.getDouble(0); - case TIMESTAMP: - case BOOLEAN: - case DATE: - case STRING: - case TEXT: - case BLOB: - case OBJECT: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + double value = rowReader.read(row); + return value >= lower && upper >= value; } @Override public void transform(Column[] columns, ColumnBuilder builder) throws Exception { - switch (dataType) { - case INT32: - transformInt(columns, builder); - return; - case INT64: - transformLong(columns, builder); - return; - case FLOAT: - transformFloat(columns, builder); - return; - case DOUBLE: - transformDouble(columns, builder); - return; - case BLOB: - case OBJECT: - case TEXT: - case DATE: - case STRING: - case BOOLEAN: - case TIMESTAMP: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } - } - - private void transformInt(Column[] columns, ColumnBuilder builder) { - int[] inputs = columns[0].getInts(); - boolean[] isNulls = columns[0].isNull(); - - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (isNulls[i]) { - builder.appendNull(); - } else { - boolean res = inputs[i] >= lower && upper >= inputs[i]; - builder.writeBoolean(res); - } - } - } - - private void transformLong(Column[] columns, ColumnBuilder builder) { - long[] inputs = columns[0].getLongs(); - boolean[] isNulls = columns[0].isNull(); - - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (isNulls[i]) { - builder.appendNull(); - } else { - boolean res = inputs[i] >= lower && upper >= inputs[i]; - builder.writeBoolean(res); - } - } - } - - private void transformFloat(Column[] columns, ColumnBuilder builder) { - float[] inputs = columns[0].getFloats(); - boolean[] isNulls = columns[0].isNull(); - - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (isNulls[i]) { - builder.appendNull(); - } else { - boolean res = inputs[i] >= lower && upper >= inputs[i]; - builder.writeBoolean(res); - } - } - } - - private void transformDouble(Column[] columns, ColumnBuilder builder) { - double[] inputs = columns[0].getDoubles(); - boolean[] isNulls = columns[0].isNull(); - - int count = columns[0].getPositionCount(); + Column column = columns[0]; + boolean[] isNulls = column.isNull(); + int count = column.getPositionCount(); for (int i = 0; i < count; i++) { if (isNulls[i]) { builder.appendNull(); } else { - boolean res = inputs[i] >= lower && upper >= inputs[i]; - builder.writeBoolean(res); + double value = columnReader.read(column, i); + builder.writeBoolean(value >= lower && upper >= value); } } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java index 9665930c1992..06a54e1c9564 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java @@ -31,7 +31,6 @@ import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; import org.apache.iotdb.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy; import org.apache.iotdb.udf.api.exception.UDFException; -import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; import org.apache.iotdb.udf.api.exception.UDFParameterNotValidException; import org.apache.iotdb.udf.api.type.Type; @@ -61,6 +60,7 @@ enum AccessStrategy { protected AccessStrategy accessStrategy; protected TSDataType dataType; + private TypeServices.NumericWindowTransformer windowTransformer; public static final String WINDOW_SIZE_KEY = "windowSize"; public static final String TIME_INTERVAL_KEY = "timeInterval"; @@ -103,6 +103,9 @@ public void validate(UDFParameterValidator validator) throws UDFException { @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws MetadataException { + windowTransformer = + TypeServices.M4_WINDOW_TRANSFORMER_SERVICE.call( + UDFDataTypeTransformer.transformUDFDataTypeToReadType(parameters.getDataType(0))); // set data type configurations.setOutputDataType(UDFDataTypeTransformer.transformToUDFDataType(dataType)); @@ -127,36 +130,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(RowWindow rowWindow, PointCollector collector) throws UDFException, IOException { - switch (dataType) { - case INT32: - transformInt(rowWindow, collector); - break; - case INT64: - transformLong(rowWindow, collector); - break; - case FLOAT: - transformFloat(rowWindow, collector); - break; - case DOUBLE: - transformDouble(rowWindow, collector); - break; - case BLOB: - case OBJECT: - case DATE: - case STRING: - case TIMESTAMP: - case BOOLEAN: - case TEXT: - default: - // This will not happen - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + windowTransformer.transform(this, rowWindow, collector); } public void transformInt(RowWindow rowWindow, PointCollector collector) throws IOException { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java index 947cbdb2c013..08179e24c54d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java @@ -48,6 +48,8 @@ protected interface Transformer { protected Transformer transformer; protected TSDataType dataType; + private TypeServices.NumericRowReader rowReader; + private TypeServices.NumericColumnReader columnReader; @Override public void validate(UDFParameterValidator validator) throws UDFException { @@ -60,6 +62,10 @@ public void validate(UDFParameterValidator validator) throws UDFException { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws MetadataException { dataType = UDFDataTypeTransformer.transformToTsDataType(parameters.getDataType(0)); + org.apache.tsfile.read.common.type.Type type = + org.apache.tsfile.read.common.type.Type.fromTsDataType(dataType); + rowReader = TypeServices.NUMERIC_ROW_READER_SERVICE.call(type); + columnReader = TypeServices.NUMERIC_COLUMN_READER_SERVICE.call(type); configurations .setAccessStrategy(new MappableRowByRowAccessStrategy()) .setOutputDataType(Type.DOUBLE); @@ -71,37 +77,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws UDFInputSeriesDataTypeNotValidException, IOException { - long time = row.getTime(); - switch (dataType) { - case INT32: - collector.putDouble(time, transformer.transform(row.getInt(0))); - break; - case INT64: - collector.putDouble(time, transformer.transform(row.getLong(0))); - break; - case FLOAT: - collector.putDouble(time, transformer.transform(row.getFloat(0))); - break; - case DOUBLE: - collector.putDouble(time, transformer.transform(row.getDouble(0))); - break; - case BOOLEAN: - case TEXT: - case STRING: - case TIMESTAMP: - case DATE: - case BLOB: - case OBJECT: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + collector.putDouble(row.getTime(), transformer.transform(rowReader.read(row))); } @Override @@ -109,120 +85,19 @@ public Object transform(Row row) throws IOException { if (row.isNull(0)) { return null; } - switch (dataType) { - case INT32: - return transformer.transform(row.getInt(0)); - case INT64: - return transformer.transform(row.getLong(0)); - case FLOAT: - return transformer.transform(row.getFloat(0)); - case DOUBLE: - return transformer.transform(row.getDouble(0)); - case DATE: - case BLOB: - case OBJECT: - case STRING: - case TIMESTAMP: - case TEXT: - case BOOLEAN: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + return transformer.transform(rowReader.read(row)); } @Override public void transform(Column[] columns, ColumnBuilder builder) throws Exception { - switch (dataType) { - case INT32: - transformInt(columns, builder); - return; - case INT64: - transformLong(columns, builder); - return; - case FLOAT: - transformFloat(columns, builder); - return; - case DOUBLE: - transformDouble(columns, builder); - return; - case TEXT: - case BOOLEAN: - case STRING: - case TIMESTAMP: - case BLOB: - case OBJECT: - case DATE: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } - } - - private void transformInt(Column[] columns, ColumnBuilder builder) { - int[] inputs = columns[0].getInts(); - boolean[] isNulls = columns[0].isNull(); - - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (isNulls[i]) { - builder.appendNull(); - } else { - builder.writeDouble(transformer.transform(inputs[i])); - } - } - } - - private void transformLong(Column[] columns, ColumnBuilder builder) { - long[] inputs = columns[0].getLongs(); - boolean[] isNulls = columns[0].isNull(); - - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (isNulls[i]) { - builder.appendNull(); - } else { - builder.writeDouble(transformer.transform(inputs[i])); - } - } - } - - private void transformFloat(Column[] columns, ColumnBuilder builder) { - float[] inputs = columns[0].getFloats(); - boolean[] isNulls = columns[0].isNull(); - - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (isNulls[i]) { - builder.appendNull(); - } else { - builder.writeDouble(transformer.transform(inputs[i])); - } - } - } - - private void transformDouble(Column[] columns, ColumnBuilder builder) { - double[] inputs = columns[0].getDoubles(); - boolean[] isNulls = columns[0].isNull(); - - int count = columns[0].getPositionCount(); + Column column = columns[0]; + boolean[] isNulls = column.isNull(); + int count = column.getPositionCount(); for (int i = 0; i < count; i++) { if (isNulls[i]) { builder.appendNull(); } else { - builder.writeDouble(transformer.transform(inputs[i])); + builder.writeDouble(transformer.transform(columnReader.read(column, i))); } } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java index 077db957f6e2..3847e8638b40 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java @@ -19,59 +19,27 @@ package org.apache.iotdb.commons.udf.builtin; -import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; -import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.read.common.type.service.TypeService; import java.io.IOException; public class UDTFNonNegativeDerivative extends UDTFDerivative { + @Override + protected TypeService derivativeOperatorService() { + return TypeServices.NON_NEGATIVE_DERIVATIVE_OPERATOR_SERVICE; + } + @Override protected void doTransform(Row row, PointCollector collector) throws UDFInputSeriesDataTypeNotValidException, IOException { long currentTime = row.getTime(); double timeDelta = (double) currentTime - previousTime; - switch (dataType) { - case INT32: - int currentInt = row.getInt(0); - collector.putDouble(currentTime, Math.abs(currentInt - previousInt) / timeDelta); - previousInt = currentInt; - break; - case INT64: - long currentLong = row.getLong(0); - collector.putDouble(currentTime, Math.abs(currentLong - previousLong) / timeDelta); - previousLong = currentLong; - break; - case FLOAT: - float currentFloat = row.getFloat(0); - collector.putDouble(currentTime, Math.abs(currentFloat - previousFloat) / timeDelta); - previousFloat = currentFloat; - break; - case DOUBLE: - double currentDouble = row.getDouble(0); - collector.putDouble(currentTime, Math.abs(currentDouble - previousDouble) / timeDelta); - previousDouble = currentDouble; - break; - case DATE: - case BLOB: - case OBJECT: - case STRING: - case TIMESTAMP: - case BOOLEAN: - case TEXT: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + derivativeOperator.apply(this, currentTime, row, collector, timeDelta); previousTime = currentTime; } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java index fcb68817f153..2cafcf446c38 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java @@ -19,57 +19,24 @@ package org.apache.iotdb.commons.udf.builtin; -import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; -import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.read.common.type.service.TypeService; import java.io.IOException; public class UDTFNonNegativeValueDifference extends UDTFValueDifference { + @Override + protected TypeService valueDifferenceOperatorService() { + return TypeServices.NON_NEGATIVE_VALUE_DIFFERENCE_OPERATOR_SERVICE; + } + @Override protected void doTransform(Row row, PointCollector collector) throws UDFInputSeriesDataTypeNotValidException, IOException { - long time = row.getTime(); - switch (dataType) { - case INT32: - int currentInt = row.getInt(0); - collector.putInt(time, Math.abs(currentInt - previousInt)); - previousInt = currentInt; - break; - case INT64: - long currentLong = row.getLong(0); - collector.putLong(time, Math.abs(currentLong - previousLong)); - previousLong = currentLong; - break; - case FLOAT: - float currentFloat = row.getFloat(0); - collector.putFloat(time, Math.abs(currentFloat - previousFloat)); - previousFloat = currentFloat; - break; - case DOUBLE: - double currentDouble = row.getDouble(0); - collector.putDouble(time, Math.abs(currentDouble - previousDouble)); - previousDouble = currentDouble; - break; - case BOOLEAN: - case TEXT: - case STRING: - case TIMESTAMP: - case BLOB: - case OBJECT: - case DATE: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + valueDifferenceOperator.apply(this, row.getTime(), row, collector); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java index b0c95bd0d23b..2095d07172ec 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java @@ -29,12 +29,10 @@ import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.MappableRowByRowAccessStrategy; import org.apache.iotdb.udf.api.exception.UDFException; -import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; import org.apache.iotdb.udf.api.type.Type; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.enums.TSDataType; import java.io.IOException; @@ -42,7 +40,8 @@ public class UDTFOnOff implements UDTF { protected double threshold; - private TSDataType dataType; + private TypeServices.NumericRowReader rowReader; + private TypeServices.NumericColumnReader columnReader; @Override public void validate(UDFParameterValidator validator) throws UDFException { @@ -56,45 +55,19 @@ public void validate(UDFParameterValidator validator) throws UDFException { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws MetadataException { threshold = parameters.getDouble("threshold"); - dataType = UDFDataTypeTransformer.transformToTsDataType(parameters.getDataType(0)); + org.apache.tsfile.read.common.type.Type type = + UDFDataTypeTransformer.transformUDFDataTypeToReadType(parameters.getDataType(0)); + rowReader = TypeServices.NUMERIC_ROW_READER_SERVICE.call(type); + columnReader = TypeServices.NUMERIC_COLUMN_READER_SERVICE.call(type); configurations .setAccessStrategy(new MappableRowByRowAccessStrategy()) .setOutputDataType(Type.BOOLEAN); } @Override - public void transform(Row row, PointCollector collector) - throws UDFInputSeriesDataTypeNotValidException, IOException { + public void transform(Row row, PointCollector collector) throws IOException { long time = row.getTime(); - switch (dataType) { - case INT32: - collector.putBoolean(time, row.getInt(0) >= threshold); - break; - case INT64: - collector.putBoolean(time, (row.getLong(0) >= threshold)); - break; - case FLOAT: - collector.putBoolean(time, (row.getFloat(0) >= threshold)); - break; - case DOUBLE: - collector.putBoolean(time, (row.getDouble(0) >= threshold)); - break; - case DATE: - case BLOB: - case STRING: - case TIMESTAMP: - case TEXT: - case BOOLEAN: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + collector.putBoolean(time, rowReader.read(row) >= threshold); } @Override @@ -102,119 +75,19 @@ public Object transform(Row row) throws IOException { if (row.isNull(0)) { return null; } - switch (dataType) { - case INT32: - return row.getInt(0) >= threshold; - case INT64: - return row.getLong(0) >= threshold; - case FLOAT: - return row.getFloat(0) >= threshold; - case DOUBLE: - return row.getDouble(0) >= threshold; - case TEXT: - case BOOLEAN: - case STRING: - case TIMESTAMP: - case BLOB: - case DATE: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + return rowReader.read(row) >= threshold; } @Override public void transform(Column[] columns, ColumnBuilder builder) throws Exception { - switch (dataType) { - case INT32: - transformInt(columns, builder); - return; - case INT64: - transformLong(columns, builder); - return; - case FLOAT: - transformFloat(columns, builder); - return; - case DOUBLE: - transformDouble(columns, builder); - return; - case BLOB: - case OBJECT: - case DATE: - case STRING: - case TIMESTAMP: - case BOOLEAN: - case TEXT: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } - } - - private void transformInt(Column[] columns, ColumnBuilder builder) { - int[] inputs = columns[0].getInts(); - boolean[] isNulls = columns[0].isNull(); - - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (isNulls[i]) { - builder.appendNull(); - } else { - builder.writeBoolean(inputs[i] >= threshold); - } - } - } - - private void transformLong(Column[] columns, ColumnBuilder builder) { - long[] inputs = columns[0].getLongs(); - boolean[] isNulls = columns[0].isNull(); - - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (isNulls[i]) { - builder.appendNull(); - } else { - builder.writeBoolean(inputs[i] >= threshold); - } - } - } - - private void transformFloat(Column[] columns, ColumnBuilder builder) { - float[] inputs = columns[0].getFloats(); - boolean[] isNulls = columns[0].isNull(); - - int count = columns[0].getPositionCount(); - for (int i = 0; i < count; i++) { - if (isNulls[i]) { - builder.appendNull(); - } else { - builder.writeBoolean(inputs[i] >= threshold); - } - } - } - - private void transformDouble(Column[] columns, ColumnBuilder builder) { - double[] inputs = columns[0].getDoubles(); - boolean[] isNulls = columns[0].isNull(); - - int count = columns[0].getPositionCount(); + Column column = columns[0]; + boolean[] isNulls = column.isNull(); + int count = column.getPositionCount(); for (int i = 0; i < count; i++) { if (isNulls[i]) { builder.appendNull(); } else { - builder.writeBoolean(inputs[i] >= threshold); + builder.writeBoolean(columnReader.read(column, i) >= threshold); } } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java index a888c998d8d8..d4c4e2bb6de2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java @@ -49,6 +49,8 @@ public abstract class UDTFSelectK implements UDTF { protected PriorityQueue> floatPQ; protected PriorityQueue> doublePQ; protected PriorityQueue> stringPQ; + private TypeServices.SelectKRowTransformer rowTransformer; + private TypeServices.SelectKTerminator terminator; @Override public void validate(UDFParameterValidator validator) throws UDFException { @@ -77,6 +79,10 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati k = parameters.getInt("k"); dataType = UDFDataTypeTransformer.transformToTsDataType(parameters.getDataType(0)); constructPQ(); + org.apache.tsfile.read.common.type.Type type = + UDFDataTypeTransformer.transformUDFDataTypeToReadType(parameters.getDataType(0)); + rowTransformer = TypeServices.SELECT_K_ROW_TRANSFORMER_SERVICE.call(type); + terminator = TypeServices.SELECT_K_TERMINATOR_SERVICE.call(type); configurations .setAccessStrategy(new RowByRowAccessStrategy()) .setOutputDataType(UDFDataTypeTransformer.transformToUDFDataType(dataType)); @@ -87,42 +93,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws UDFInputSeriesDataTypeNotValidException, IOException { - switch (dataType) { - case INT32: - case DATE: - transformInt(row.getTime(), row.getInt(0)); - break; - case INT64: - case TIMESTAMP: - transformLong(row.getTime(), row.getLong(0)); - break; - case FLOAT: - transformFloat(row.getTime(), row.getFloat(0)); - break; - case DOUBLE: - transformDouble(row.getTime(), row.getDouble(0)); - break; - case TEXT: - case STRING: - transformString(row.getTime(), row.getString(0)); - break; - case BLOB: - case OBJECT: - case BOOLEAN: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE, - Type.TEXT, - Type.DATE, - Type.TIMESTAMP, - Type.STRING); - } + rowTransformer.transform(this, row); } protected abstract void transformInt(long time, int value); @@ -138,64 +109,41 @@ public void transform(Row row, PointCollector collector) @Override public void terminate(PointCollector collector) throws UDFInputSeriesDataTypeNotValidException, IOException { - switch (dataType) { - case INT32: - case DATE: - for (Pair pair : - intPQ.stream().sorted(Comparator.comparing(p -> p.left)).collect(Collectors.toList())) { - collector.putInt(pair.left, pair.right); - } - break; - case INT64: - case TIMESTAMP: - for (Pair pair : - longPQ.stream() - .sorted(Comparator.comparing(p -> p.left)) - .collect(Collectors.toList())) { - collector.putLong(pair.left, pair.right); - } - break; - case FLOAT: - for (Pair pair : - floatPQ.stream() - .sorted(Comparator.comparing(p -> p.left)) - .collect(Collectors.toList())) { - collector.putFloat(pair.left, pair.right); - } - break; - case DOUBLE: - for (Pair pair : - doublePQ.stream() - .sorted(Comparator.comparing(p -> p.left)) - .collect(Collectors.toList())) { - collector.putDouble(pair.left, pair.right); - } - break; - case TEXT: - case STRING: - for (Pair pair : - stringPQ.stream() - .sorted(Comparator.comparing(p -> p.left)) - .collect(Collectors.toList())) { - collector.putString(pair.left, pair.right); - } - break; - case BLOB: - case OBJECT: - case BOOLEAN: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE, - Type.TEXT, - Type.DATE, - Type.TIMESTAMP, - Type.STRING); + terminator.terminate(this, collector); + } + + void terminateInt(PointCollector collector) throws IOException { + for (Pair pair : + intPQ.stream().sorted(Comparator.comparing(p -> p.left)).collect(Collectors.toList())) { + collector.putInt(pair.left, pair.right); + } + } + + void terminateLong(PointCollector collector) throws IOException { + for (Pair pair : + longPQ.stream().sorted(Comparator.comparing(p -> p.left)).collect(Collectors.toList())) { + collector.putLong(pair.left, pair.right); + } + } + + void terminateFloat(PointCollector collector) throws IOException { + for (Pair pair : + floatPQ.stream().sorted(Comparator.comparing(p -> p.left)).collect(Collectors.toList())) { + collector.putFloat(pair.left, pair.right); + } + } + + void terminateDouble(PointCollector collector) throws IOException { + for (Pair pair : + doublePQ.stream().sorted(Comparator.comparing(p -> p.left)).collect(Collectors.toList())) { + collector.putDouble(pair.left, pair.right); + } + } + + void terminateString(PointCollector collector) throws IOException { + for (Pair pair : + stringPQ.stream().sorted(Comparator.comparing(p -> p.left)).collect(Collectors.toList())) { + collector.putString(pair.left, pair.right); } } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java index b8d94308ed96..fb37c91d9f4a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java @@ -19,10 +19,6 @@ package org.apache.iotdb.commons.udf.builtin; -import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; -import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; -import org.apache.iotdb.udf.api.type.Type; - import org.apache.tsfile.utils.Pair; import java.util.Comparator; @@ -31,43 +27,30 @@ public class UDTFTopK extends UDTFSelectK { @Override - protected void constructPQ() throws UDFInputSeriesDataTypeNotValidException { - switch (dataType) { - case INT32: - case DATE: - intPQ = new PriorityQueue<>(k, Comparator.comparing(o -> o.right)); - break; - case INT64: - case TIMESTAMP: - longPQ = new PriorityQueue<>(k, Comparator.comparing(o -> o.right)); - break; - case FLOAT: - floatPQ = new PriorityQueue<>(k, Comparator.comparing(o -> o.right)); - break; - case DOUBLE: - doublePQ = new PriorityQueue<>(k, Comparator.comparing(o -> o.right)); - break; - case TEXT: - case STRING: - stringPQ = new PriorityQueue<>(k, Comparator.comparing(o -> o.right)); - break; - case BOOLEAN: - case BLOB: - case OBJECT: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE, - Type.TEXT, - Type.DATE, - Type.TIMESTAMP, - Type.STRING); - } + protected void constructPQ() { + TypeServices.TOP_K_QUEUE_CONSTRUCTOR_SERVICE + .call(org.apache.tsfile.read.common.type.Type.fromTsDataType(dataType)) + .construct(this); + } + + void initializeIntQueue() { + intPQ = new PriorityQueue<>(k, Comparator.comparing(o -> o.right)); + } + + void initializeLongQueue() { + longPQ = new PriorityQueue<>(k, Comparator.comparing(o -> o.right)); + } + + void initializeFloatQueue() { + floatPQ = new PriorityQueue<>(k, Comparator.comparing(o -> o.right)); + } + + void initializeDoubleQueue() { + doublePQ = new PriorityQueue<>(k, Comparator.comparing(o -> o.right)); + } + + void initializeStringQueue() { + stringPQ = new PriorityQueue<>(k, Comparator.comparing(o -> o.right)); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueDifference.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueDifference.java index 9804e1ce374a..81d8cc78aa7f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueDifference.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueDifference.java @@ -28,14 +28,26 @@ import org.apache.iotdb.udf.api.customizer.strategy.RowByRowAccessStrategy; import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; + import java.io.IOException; public abstract class UDTFValueDifference extends UDTFValueTrend { + protected TypeServices.ValueDifferenceOperator valueDifferenceOperator; + + protected TypeService valueDifferenceOperatorService() { + return TypeServices.VALUE_DIFFERENCE_OPERATOR_SERVICE; + } + @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws MetadataException { dataType = UDFDataTypeTransformer.transformToTsDataType(parameters.getDataType(0)); + Type type = Type.fromTsDataType(dataType); + previousValueReader = TypeServices.VALUE_TREND_READER_SERVICE.call(type); + valueDifferenceOperator = valueDifferenceOperatorService().call(type); configurations .setAccessStrategy(new RowByRowAccessStrategy()) .setOutputDataType(UDFDataTypeTransformer.transformToUDFDataType(dataType)); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java index d4487fdfd953..1f8a77188227 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java @@ -42,6 +42,7 @@ public abstract class UDTFValueTrend implements UDTF { protected double previousDouble = 0; protected TSDataType dataType; + protected TypeServices.PreviousValueReader previousValueReader; @Override public void validate(UDFParameterValidator validator) throws UDFException { @@ -52,36 +53,17 @@ public void validate(UDFParameterValidator validator) throws UDFException { protected void updatePreviousValue(Row row) throws UDFInputSeriesDataTypeNotValidException, IOException { - switch (dataType) { - case INT32: - previousInt = row.getInt(0); - break; - case INT64: - previousLong = row.getLong(0); - break; - case FLOAT: - previousFloat = row.getFloat(0); - break; - case DOUBLE: - previousDouble = row.getDouble(0); - break; - case TEXT: - case BOOLEAN: - case TIMESTAMP: - case STRING: - case DATE: - case BLOB: - case OBJECT: - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, - UDFDataTypeTransformer.transformToUDFDataType(dataType), - Type.INT32, - Type.INT64, - Type.FLOAT, - Type.DOUBLE); - } + previousValueReader.read(this, row); + } + + protected UDFInputSeriesDataTypeNotValidException invalidDataType() { + return new UDFInputSeriesDataTypeNotValidException( + 0, + UDFDataTypeTransformer.transformToUDFDataType(dataType), + Type.INT32, + Type.INT64, + Type.FLOAT, + Type.DOUBLE); } protected abstract void doTransform(Row row, PointCollector collector) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/MasterRepairUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/MasterRepairUtil.java index 79d5381f45da..5a9fe141fe15 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/MasterRepairUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/MasterRepairUtil.java @@ -20,13 +20,18 @@ import org.apache.iotdb.commons.i18n.CommonMessages; import org.apache.iotdb.udf.api.access.Row; +import org.apache.iotdb.udf.api.type.Type; import java.io.IOException; import java.math.BigDecimal; import java.util.ArrayList; import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; public class MasterRepairUtil { + private static final Set NUMERIC_TYPES = + EnumSet.of(Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); private final ArrayList> td = new ArrayList<>(); private final ArrayList> tdCleaned = new ArrayList<>(); private final ArrayList> md = new ArrayList<>(); @@ -90,34 +95,15 @@ public void addRow(Row row) throws Exception { } public static double getValueAsDouble(Row row, int index) throws Exception { - double ans; try { - switch (row.getDataType(index)) { - case INT32: - ans = row.getInt(index); - break; - case INT64: - ans = row.getLong(index); - break; - case FLOAT: - ans = row.getFloat(index); - break; - case DOUBLE: - ans = row.getDouble(index); - break; - case DATE: - case BLOB: - case STRING: - case TIMESTAMP: - case BOOLEAN: - case TEXT: - default: - throw new Exception(CommonMessages.VALUE_NOT_NUMERIC); + // DATE and TIMESTAMP have numeric storage but are not repairable numeric measurements. + if (!NUMERIC_TYPES.contains(row.getDataType(index))) { + throw new Exception(CommonMessages.VALUE_NOT_NUMERIC); } + return row.getDouble(index); } catch (IOException e) { throw new Exception(CommonMessages.FAIL_TO_GET_DATA_TYPE_IN_ROW + row.getTime(), e); } - return ans; } public void buildKDTree() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java index 163e9b4135a5..e8f826b8e557 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java @@ -23,17 +23,6 @@ import org.apache.iotdb.udf.api.type.Type; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.type.BinaryType; -import org.apache.tsfile.read.common.type.BlobType; -import org.apache.tsfile.read.common.type.BooleanType; -import org.apache.tsfile.read.common.type.DateType; -import org.apache.tsfile.read.common.type.DoubleType; -import org.apache.tsfile.read.common.type.FloatType; -import org.apache.tsfile.read.common.type.IntType; -import org.apache.tsfile.read.common.type.LongType; -import org.apache.tsfile.read.common.type.ObjectType; -import org.apache.tsfile.read.common.type.StringType; -import org.apache.tsfile.read.common.type.TimestampType; import java.util.List; import java.util.stream.Collectors; @@ -48,7 +37,14 @@ public static TSDataType transformToTsDataType(Type type) { } public static Type transformToUDFDataType(TSDataType tsDataType) { - return tsDataType == null ? null : getUDFDataType(tsDataType.getType()); + if (tsDataType == null) { + return null; + } + try { + return Type.valueOf(tsDataType.getType()); + } catch (IllegalArgumentException e) { + throw invalidInput(tsDataType, e); + } } public static List transformToUDFDataTypeList(List tsDataTypeList) { @@ -63,31 +59,10 @@ public static Type transformReadTypeToUDFDataType(org.apache.tsfile.read.common. if (type == null) { return null; } - switch (type.getTypeEnum()) { - case BOOLEAN: - return Type.BOOLEAN; - case INT32: - return Type.INT32; - case INT64: - return Type.INT64; - case FLOAT: - return Type.FLOAT; - case DOUBLE: - return Type.DOUBLE; - case TEXT: - return Type.TEXT; - case TIMESTAMP: - return Type.TIMESTAMP; - case DATE: - return Type.DATE; - case BLOB: - return Type.BLOB; - case STRING: - return Type.STRING; - case OBJECT: - return Type.OBJECT; - default: - throw new IllegalArgumentException(SchemaMessages.SCHEMA_INVALID_INPUT + type); + try { + return transformToUDFDataType(TSDataType.valueOf(type.getTypeEnum().name())); + } catch (IllegalArgumentException e) { + throw invalidInput(type, e); } } @@ -95,60 +70,12 @@ public static org.apache.tsfile.read.common.type.Type transformUDFDataTypeToRead if (type == null) { return null; } - switch (type) { - case BOOLEAN: - return BooleanType.BOOLEAN; - case INT32: - return IntType.INT32; - case DATE: - return DateType.DATE; - case INT64: - return LongType.INT64; - case TIMESTAMP: - return TimestampType.TIMESTAMP; - case FLOAT: - return FloatType.FLOAT; - case DOUBLE: - return DoubleType.DOUBLE; - case TEXT: - return BinaryType.TEXT; - case BLOB: - return BlobType.BLOB; - case STRING: - return StringType.STRING; - case OBJECT: - return ObjectType.OBJECT; - default: - throw new IllegalArgumentException(SchemaMessages.SCHEMA_INVALID_INPUT + type); - } + return org.apache.tsfile.read.common.type.Type.fromTsDataType( + TSDataType.getTsDataType(type.getType())); } - private static Type getUDFDataType(byte type) { - switch (type) { - case 0: - return Type.BOOLEAN; - case 1: - return Type.INT32; - case 2: - return Type.INT64; - case 3: - return Type.FLOAT; - case 4: - return Type.DOUBLE; - case 5: - return Type.TEXT; - case 8: - return Type.TIMESTAMP; - case 9: - return Type.DATE; - case 10: - return Type.BLOB; - case 11: - return Type.STRING; - case 12: - return Type.OBJECT; - default: - throw new IllegalArgumentException(SchemaMessages.SCHEMA_INVALID_INPUT + type); - } + private static IllegalArgumentException invalidInput( + Object type, IllegalArgumentException cause) { + return new IllegalArgumentException(SchemaMessages.SCHEMA_INVALID_INPUT + type, cause); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java index bbc3a101edbb..6e1ded9e40ba 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java @@ -28,9 +28,8 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.BatchData.BatchDataType; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.utils.TsPrimitiveType; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; @@ -160,187 +159,20 @@ public static BatchData deserializeBatchData(ByteBuffer buffer) { int length = buffer.getInt(); TSDataType dataType = TSDataType.values()[buffer.get()]; BatchData batchData = BatchDataType.deserialize(buffer.get(), dataType); - switch (dataType) { - case DATE: - case INT32: - for (int i = 0; i < length; i++) { - batchData.putInt(buffer.getLong(), buffer.getInt()); - } - break; - case TIMESTAMP: - case INT64: - for (int i = 0; i < length; i++) { - batchData.putLong(buffer.getLong(), buffer.getLong()); - } - break; - case BLOB: - case OBJECT: - case STRING: - case TEXT: - for (int i = 0; i < length; i++) { - long time = buffer.getLong(); - int len = buffer.getInt(); - byte[] bytes = new byte[len]; - buffer.get(bytes); - batchData.putBinary(time, new Binary(bytes)); - } - break; - case FLOAT: - for (int i = 0; i < length; i++) { - batchData.putFloat(buffer.getLong(), buffer.getFloat()); - } - break; - case DOUBLE: - for (int i = 0; i < length; i++) { - batchData.putDouble(buffer.getLong(), buffer.getDouble()); - } - break; - case BOOLEAN: - for (int i = 0; i < length; i++) { - batchData.putBoolean(buffer.getLong(), buffer.get() == 1); - } - break; - case VECTOR: - for (int i = 0; i < length; i++) { - long time = buffer.getLong(); - int valuesLength = buffer.getInt(); - TsPrimitiveType[] values = new TsPrimitiveType[valuesLength]; - for (int j = 0; j < valuesLength; j++) { - boolean notNull = (buffer.get() == 1); - if (notNull) { - switch (TSDataType.values()[buffer.get()]) { - case BOOLEAN: - values[j] = new TsPrimitiveType.TsBoolean(buffer.get() == 1); - break; - case DOUBLE: - values[j] = new TsPrimitiveType.TsDouble(buffer.getDouble()); - break; - case FLOAT: - values[j] = new TsPrimitiveType.TsFloat(buffer.getFloat()); - break; - case BLOB: - case OBJECT: - case STRING: - case TEXT: - int len = buffer.getInt(); - byte[] bytes = new byte[len]; - buffer.get(bytes); - values[j] = new TsPrimitiveType.TsBinary(new Binary(bytes)); - break; - case TIMESTAMP: - case INT64: - values[j] = new TsPrimitiveType.TsLong(buffer.getLong()); - break; - case DATE: - case INT32: - values[j] = new TsPrimitiveType.TsInt(buffer.getInt()); - break; - default: - break; - } - } - } - batchData.putVector(time, values); - } - break; - } + Type.fromTsDataType(dataType).deserialize(buffer, batchData, length); batchData.resetBatchData(); return batchData; } - private static void serializeTextTVPairs( - List timeValuePairs, DataOutputStream dataOutputStream) throws IOException { - for (TimeValuePair timeValuePair : timeValuePairs) { - dataOutputStream.writeLong(timeValuePair.getTimestamp()); - if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { - dataOutputStream.writeInt(timeValuePair.getValue().getBinary().getLength()); - dataOutputStream.write(timeValuePair.getValue().getBinary().getValues()); - } - } - } - - private static void serializeBooleanTVPairs( - List timeValuePairs, DataOutputStream dataOutputStream) throws IOException { - for (TimeValuePair timeValuePair : timeValuePairs) { - dataOutputStream.writeLong(timeValuePair.getTimestamp()); - if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { - dataOutputStream.writeBoolean(timeValuePair.getValue().getBoolean()); - } - } - } - - private static void serializeIntTVPairs( - List timeValuePairs, DataOutputStream dataOutputStream) throws IOException { - for (TimeValuePair timeValuePair : timeValuePairs) { - dataOutputStream.writeLong(timeValuePair.getTimestamp()); - if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { - dataOutputStream.writeInt(timeValuePair.getValue().getInt()); - } - } - } - - private static void serializeLongTVPairs( - List timeValuePairs, DataOutputStream dataOutputStream) throws IOException { - for (TimeValuePair timeValuePair : timeValuePairs) { - dataOutputStream.writeLong(timeValuePair.getTimestamp()); - if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { - dataOutputStream.writeLong(timeValuePair.getValue().getLong()); - } - } - } - - private static void serializeFloatTVPairs( - List timeValuePairs, DataOutputStream dataOutputStream) throws IOException { - for (TimeValuePair timeValuePair : timeValuePairs) { - dataOutputStream.writeLong(timeValuePair.getTimestamp()); - if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { - dataOutputStream.writeFloat(timeValuePair.getValue().getFloat()); - } - } - } - - private static void serializeDoubleTVPairs( - List timeValuePairs, DataOutputStream dataOutputStream) throws IOException { - for (TimeValuePair timeValuePair : timeValuePairs) { - dataOutputStream.writeLong(timeValuePair.getTimestamp()); - if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { - dataOutputStream.writeDouble(timeValuePair.getValue().getDouble()); - } - } - } - public static void serializeTVPairs( List timeValuePairs, DataOutputStream dataOutputStream) { try { TSDataType dataType = timeValuePairs.get(0).getValue().getDataType(); dataOutputStream.write(dataType.ordinal()); dataOutputStream.writeInt(timeValuePairs.size()); - switch (timeValuePairs.get(0).getValue().getDataType()) { - case BLOB: - case OBJECT: - case STRING: - case TEXT: - serializeTextTVPairs(timeValuePairs, dataOutputStream); - break; - case BOOLEAN: - serializeBooleanTVPairs(timeValuePairs, dataOutputStream); - break; - case TIMESTAMP: - case INT64: - serializeLongTVPairs(timeValuePairs, dataOutputStream); - break; - case DATE: - case INT32: - serializeIntTVPairs(timeValuePairs, dataOutputStream); - break; - case FLOAT: - serializeFloatTVPairs(timeValuePairs, dataOutputStream); - break; - case DOUBLE: - serializeDoubleTVPairs(timeValuePairs, dataOutputStream); - break; - default: - break; + Type type = Type.fromTsDataType(dataType); + for (TimeValuePair timeValuePair : timeValuePairs) { + serializeTVPair(type, timeValuePair, dataOutputStream); } } catch (IOException ignored) { // unreachable @@ -355,132 +187,19 @@ public static void serializeTVPair( TSDataType dataType = timeValuePair.getValue().getDataType(); try { dataOutputStream.write(dataType.ordinal()); - switch (dataType) { - case STRING: - case BLOB: - case OBJECT: - case TEXT: - dataOutputStream.writeLong(timeValuePair.getTimestamp()); - if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { - dataOutputStream.writeInt(timeValuePair.getValue().getBinary().getLength()); - dataOutputStream.write(timeValuePair.getValue().getBinary().getValues()); - } - break; - case BOOLEAN: - dataOutputStream.writeLong(timeValuePair.getTimestamp()); - if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { - dataOutputStream.writeBoolean(timeValuePair.getValue().getBoolean()); - } - break; - case TIMESTAMP: - case INT64: - dataOutputStream.writeLong(timeValuePair.getTimestamp()); - if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { - dataOutputStream.writeLong(timeValuePair.getValue().getLong()); - } - break; - case DATE: - case INT32: - dataOutputStream.writeLong(timeValuePair.getTimestamp()); - if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { - dataOutputStream.writeInt(timeValuePair.getValue().getInt()); - } - break; - case FLOAT: - dataOutputStream.writeLong(timeValuePair.getTimestamp()); - if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { - dataOutputStream.writeFloat(timeValuePair.getValue().getFloat()); - } - break; - case DOUBLE: - dataOutputStream.writeLong(timeValuePair.getTimestamp()); - if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { - dataOutputStream.writeDouble(timeValuePair.getValue().getDouble()); - } - break; - default: - break; - } + serializeTVPair(Type.fromTsDataType(dataType), timeValuePair, dataOutputStream); } catch (IOException e) { // unreachable } } - private static void deserializeDoubleTVPairs( - ByteBuffer buffer, List ret, int size, TSDataType dataType) { - for (int i = 0; i < size; i++) { - long time = buffer.getLong(); - TimeValuePair pair = - time != Long.MIN_VALUE - ? new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.getDouble())) - : new TimeValuePair(time, null); - ret.add(pair); - } - } - - private static void deserializeFloatTVPairs( - ByteBuffer buffer, List ret, int size, TSDataType dataType) { - for (int i = 0; i < size; i++) { - long time = buffer.getLong(); - TimeValuePair pair = - time != Long.MIN_VALUE - ? new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.getFloat())) - : new TimeValuePair(time, null); - ret.add(pair); - } - } - - private static void deserializeIntTVPairs( - ByteBuffer buffer, List ret, int size, TSDataType dataType) { - for (int i = 0; i < size; i++) { - long time = buffer.getLong(); - TimeValuePair pair = - time != Long.MIN_VALUE - ? new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.getInt())) - : new TimeValuePair(time, null); - ret.add(pair); - } - } - - private static void deserializeLongTVPairs( - ByteBuffer buffer, List ret, int size, TSDataType dataType) { - for (int i = 0; i < size; i++) { - long time = buffer.getLong(); - TimeValuePair pair = - time != Long.MIN_VALUE - ? new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.getLong())) - : new TimeValuePair(time, null); - ret.add(pair); - } - } - - private static void deserializeBooleanTVPairs( - ByteBuffer buffer, List ret, int size, TSDataType dataType) { - for (int i = 0; i < size; i++) { - long time = buffer.getLong(); - TimeValuePair pair = - time != Long.MIN_VALUE - ? new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.get() == 1)) - : new TimeValuePair(time, null); - ret.add(pair); - } - } - - private static void deserializeTextTVPairs( - ByteBuffer buffer, List ret, int size, TSDataType dataType) { - for (int i = 0; i < size; i++) { - long time = buffer.getLong(); - TimeValuePair pair; - if (time != Long.MIN_VALUE) { - int bytesLen = buffer.getInt(); - byte[] bytes = new byte[bytesLen]; - buffer.get(bytes); - TsPrimitiveType primitiveType = TsPrimitiveType.getByType(dataType, new Binary(bytes)); - pair = new TimeValuePair(time, primitiveType); - } else { - pair = new TimeValuePair(time, null); - } - ret.add(pair); + private static void serializeTVPair( + Type type, TimeValuePair timeValuePair, DataOutputStream dataOutputStream) + throws IOException { + if (timeValuePair.getTimestamp() == Long.MIN_VALUE) { + dataOutputStream.writeLong(Long.MIN_VALUE); + } else { + type.serialize(timeValuePair, dataOutputStream); } } @@ -491,32 +210,9 @@ public static List deserializeTVPairs(ByteBuffer buffer) { TSDataType dataType = TSDataType.values()[buffer.get()]; int size = buffer.getInt(); List ret = new ArrayList<>(size); - switch (dataType) { - case DOUBLE: - deserializeDoubleTVPairs(buffer, ret, size, dataType); - break; - case FLOAT: - deserializeFloatTVPairs(buffer, ret, size, dataType); - break; - case DATE: - case INT32: - deserializeIntTVPairs(buffer, ret, size, dataType); - break; - case TIMESTAMP: - case INT64: - deserializeLongTVPairs(buffer, ret, size, dataType); - break; - case BOOLEAN: - deserializeBooleanTVPairs(buffer, ret, size, dataType); - break; - case BLOB: - case OBJECT: - case STRING: - case TEXT: - deserializeTextTVPairs(buffer, ret, size, dataType); - break; - default: - break; + Type type = Type.fromTsDataType(dataType); + for (int i = 0; i < size; i++) { + ret.add(deserializeTVPair(type, buffer)); } return ret; } @@ -525,36 +221,13 @@ public static TimeValuePair deserializeTVPair(ByteBuffer buffer) { if (buffer == null || (buffer.limit() - buffer.position() == 0)) { return null; } - TSDataType dataType = TSDataType.values()[buffer.get()]; + Type type = Type.fromTsDataType(TSDataType.values()[buffer.get()]); + return deserializeTVPair(type, buffer); + } + + private static TimeValuePair deserializeTVPair(Type type, ByteBuffer buffer) { long time = buffer.getLong(); - if (time == Long.MIN_VALUE) { - return new TimeValuePair(time, null); - } - switch (dataType) { - case DOUBLE: - return new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.getDouble())); - case FLOAT: - return new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.getFloat())); - case DATE: - case INT32: - return new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.getInt())); - case TIMESTAMP: - case INT64: - return new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.getLong())); - case BOOLEAN: - return new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.get() == 1)); - case BLOB: - case OBJECT: - case STRING: - case TEXT: - int bytesLen = buffer.getInt(); - byte[] bytes = new byte[bytesLen]; - buffer.get(bytes); - TsPrimitiveType primitiveType = TsPrimitiveType.getByType(dataType, new Binary(bytes)); - return new TimeValuePair(time, primitiveType); - default: - return null; - } + return new TimeValuePair(time, time == Long.MIN_VALUE ? null : type.deserialize(buffer)); } public static void serializeObject(Object object, DataOutputStream dataOutputStream) { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/queryengine/plan/relational/type/InternalTypeManagerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/queryengine/plan/relational/type/InternalTypeManagerTest.java new file mode 100644 index 000000000000..f579d0d0f016 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/queryengine/plan/relational/type/InternalTypeManagerTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.commons.queryengine.plan.relational.type; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.RowType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.VectorType; +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; + +public class InternalTypeManagerTest { + + @Test + public void testScalarTypeConversion() { + for (final TSDataType dataType : TSDataType.values()) { + if (dataType == TSDataType.VECTOR) { + continue; + } + + final Type type = InternalTypeManager.fromTSDataType(dataType); + assertSame(Type.fromTsDataType(dataType), type); + assertEquals(dataType, InternalTypeManager.getTSDataType(type)); + } + } + + @Test + public void testSpecialTypes() { + assertNull(InternalTypeManager.getTSDataType(null)); + assertEquals( + TSDataType.BLOB, + InternalTypeManager.getTSDataType( + RowType.anonymousRow(Type.fromTsDataType(TSDataType.INT32)))); + + Assert.assertThrows( + IllegalArgumentException.class, + () -> InternalTypeManager.fromTSDataType(TSDataType.VECTOR)); + Assert.assertThrows( + IllegalArgumentException.class, () -> InternalTypeManager.getTSDataType(VectorType.VECTOR)); + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/utils/MasterRepairUtilTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/utils/MasterRepairUtilTest.java new file mode 100644 index 000000000000..45e8eeefe48d --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/utils/MasterRepairUtilTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.udf.utils; + +import org.apache.iotdb.commons.i18n.CommonMessages; +import org.apache.iotdb.udf.api.access.Row; +import org.apache.iotdb.udf.api.type.Type; +import org.apache.iotdb.udf.api.utils.RowImpl; + +import org.apache.tsfile.enums.TSDataType; +import org.junit.Test; + +import java.io.IOException; +import java.util.EnumSet; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class MasterRepairUtilTest { + + @Test + public void testNumericValues() throws Exception { + // Use real boxed values to catch casts to Double and exercise nonzero column indices. + RowImpl row = + new RowImpl( + new TSDataType[] { + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TIMESTAMP + }); + row.setRowRecord(new Object[] {-42, Long.MAX_VALUE, 0.1f, -1.25d, 123L}); + + assertEquals(-42d, MasterRepairUtil.getValueAsDouble(row, 0), 0d); + assertEquals((double) Long.MAX_VALUE, MasterRepairUtil.getValueAsDouble(row, 1), 0d); + assertEquals((double) 0.1f, MasterRepairUtil.getValueAsDouble(row, 2), 0d); + assertEquals(-1.25d, MasterRepairUtil.getValueAsDouble(row, 3), 0d); + } + + @Test + public void testRejectNonNumericTypes() throws Exception { + // In particular, DATE and TIMESTAMP must not become valid repair measurements. + for (Type type : + EnumSet.complementOf(EnumSet.of(Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE))) { + Row row = mock(Row.class); + when(row.getDataType(1)).thenReturn(type); + + Exception exception = + assertThrows(Exception.class, () -> MasterRepairUtil.getValueAsDouble(row, 1)); + + assertEquals(CommonMessages.VALUE_NOT_NUMERIC, exception.getMessage()); + verify(row, never()).getDouble(1); + } + } + + @Test + public void testWrapIOException() throws Exception { + // A failed value read must retain the original cause and row timestamp in the message. + Row row = mock(Row.class); + IOException cause = new IOException("test read failure"); + when(row.getDataType(1)).thenReturn(Type.INT32); + when(row.getDouble(1)).thenThrow(cause); + when(row.getTime()).thenReturn(123L); + + Exception exception = + assertThrows(Exception.class, () -> MasterRepairUtil.getValueAsDouble(row, 1)); + + assertSame(cause, exception.getCause()); + assertEquals(CommonMessages.FAIL_TO_GET_DATA_TYPE_IN_ROW + 123L, exception.getMessage()); + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformerTest.java new file mode 100644 index 000000000000..8289b6906cec --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformerTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.commons.udf.utils; + +import org.apache.iotdb.commons.i18n.SchemaMessages; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.RowType; +import org.apache.tsfile.read.common.type.UnknownType; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; + +public class UDFDataTypeTransformerTest { + + @Test + public void testSupportedTypesUseTypeInterfaces() { + for (Type udfType : Type.allTypes()) { + TSDataType tsDataType = TSDataType.getTsDataType(udfType.getType()); + + Assert.assertEquals(udfType, UDFDataTypeTransformer.transformToUDFDataType(tsDataType)); + org.apache.tsfile.read.common.type.Type readType = + UDFDataTypeTransformer.transformUDFDataTypeToReadType(udfType); + Assert.assertEquals(tsDataType.name(), readType.getTypeEnum().name()); + Assert.assertEquals(udfType, UDFDataTypeTransformer.transformReadTypeToUDFDataType(readType)); + } + } + + @Test + public void testNullTypesRemainNull() { + Assert.assertNull(UDFDataTypeTransformer.transformToUDFDataType(null)); + Assert.assertNull(UDFDataTypeTransformer.transformReadTypeToUDFDataType(null)); + Assert.assertNull(UDFDataTypeTransformer.transformUDFDataTypeToReadType(null)); + } + + @Test + public void testInternalTypesRemainUnsupported() { + assertInvalidInput(() -> UDFDataTypeTransformer.transformToUDFDataType(TSDataType.UNKNOWN)); + assertInvalidInput(() -> UDFDataTypeTransformer.transformToUDFDataType(TSDataType.VECTOR)); + assertInvalidInput( + () -> UDFDataTypeTransformer.transformReadTypeToUDFDataType(UnknownType.UNKNOWN)); + assertInvalidInput( + () -> + UDFDataTypeTransformer.transformReadTypeToUDFDataType( + RowType.anonymous(Collections.emptyList()))); + } + + private static void assertInvalidInput(Runnable conversion) { + IllegalArgumentException exception = + Assert.assertThrows(IllegalArgumentException.class, conversion::run); + Assert.assertTrue(exception.getMessage().startsWith(SchemaMessages.SCHEMA_INVALID_INPUT)); + } +} diff --git a/library-udf/pom.xml b/library-udf/pom.xml index ef592895cc39..8ed776c7b440 100644 --- a/library-udf/pom.xml +++ b/library-udf/pom.xml @@ -38,6 +38,11 @@ common ${tsfile.version} + + org.apache.tsfile + tsfile + ${tsfile.version} + org.apache.iotdb udf-api diff --git a/library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java b/library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java index 3b77edc96af0..459037d8333b 100644 --- a/library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java +++ b/library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java @@ -60,6 +60,10 @@ public final class LibraryUdfMessages { public static final String ALL_VALUES_ARE_NAN = "All values are NaN"; // UDFEnvelopeAnalysis + public static final String EXCEPTION_THE_PARAM_FREQUENCY_MUST_BE_GREATER_THAN_0_45820CF9 = + "The param 'frequency' must be greater than 0."; + public static final String EXCEPTION_THE_PARAM_AMPLIFICATION_MUST_BE_GREATER_THAN_OR_EQUAL_TO_1_D64050EB = + "The param 'amplification' must be greater than or equal to 1."; public static final String UNSUPPORTED_TIME_UNIT = "Unsupported time unit."; // UDTFAR @@ -100,6 +104,17 @@ public final class LibraryUdfMessages { // UDAFPatternMatch, UDAFDTWMatch public static final String UNSUPPORTED_DATATYPE = "Unsupported datatype %s"; + // ExactOrderStatistics + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_B411C29E = + "Unsupported data type: %s"; + + // UDAFQuantile + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_A8CA7BE7 = "Unsupported data type"; + public static final String EXCEPTION_SIZE_K_HAS_TO_BE_GREATER_THAN_OR_EQUAL_TO_100_C514D1C3 = + "Size K has to be greater than or equal to 100."; + public static final String EXCEPTION_RANK_HAS_TO_BE_GREATER_THAN_0_AND_LESS_THAN_OR_EQUAL_TO_1_0F16AF94 = + "rank has to be greater than 0 and less than or equal to 1."; + // LinearScale public static final String DOMAIN_START_MUST_BE_LESS_THAN_DOMAIN_END = "domainStart must be less than domainEnd"; diff --git a/library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java b/library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java index d4fdafddb29a..467bdbf30796 100644 --- a/library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java +++ b/library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java @@ -57,6 +57,9 @@ public final class LibraryUdfMessages { public static final String ALL_VALUES_ARE_NAN = "所有值均为 NaN"; // UDFEnvelopeAnalysis + public static final String EXCEPTION_THE_PARAM_FREQUENCY_MUST_BE_GREATER_THAN_0_45820CF9 = "参数 'frequency' 必须大于 0。"; + public static final String EXCEPTION_THE_PARAM_AMPLIFICATION_MUST_BE_GREATER_THAN_OR_EQUAL_TO_1_D64050EB = + "参数 'amplification' 必须大于或等于 1。"; public static final String UNSUPPORTED_TIME_UNIT = "不支持的时间单位。"; // UDTFAR @@ -91,6 +94,15 @@ public final class LibraryUdfMessages { // UDAFPatternMatch, UDAFDTWMatch public static final String UNSUPPORTED_DATATYPE = "不支持的数据类型 %s"; + // ExactOrderStatistics + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_B411C29E = "不支持的数据类型：%s"; + + // UDAFQuantile + public static final String EXCEPTION_UNSUPPORTED_DATA_TYPE_A8CA7BE7 = "不支持的数据类型"; + public static final String EXCEPTION_SIZE_K_HAS_TO_BE_GREATER_THAN_OR_EQUAL_TO_100_C514D1C3 = "K 的大小必须大于或等于 100。"; + public static final String EXCEPTION_RANK_HAS_TO_BE_GREATER_THAN_0_AND_LESS_THAN_OR_EQUAL_TO_1_0F16AF94 = + "rank 必须大于 0 且小于或等于 1。"; + // LinearScale public static final String DOMAIN_START_MUST_BE_LESS_THAN_DOMAIN_END = "domainStart 必须小于 domainEnd"; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java index 1d63d040d87f..4de6980dd742 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java @@ -21,7 +21,7 @@ import org.apache.iotdb.library.util.CircularQueue; import org.apache.iotdb.library.util.LongCircularQueue; -import org.apache.iotdb.library.util.Util; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; @@ -42,6 +42,9 @@ public class UDTFKSigma implements UDTF { private CircularQueue v; private LongCircularQueue t; private Type dataType; + private TypeServices.RowValueReader valueReader; + private TypeServices.NumericRowReader numericReader; + private TypeServices.RowValueWriter valueWriter; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -66,6 +69,9 @@ public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConf .setOutputDataType(udfParameters.getDataType(0)); this.multipleK = udfParameters.getDoubleOrDefault("k", 3); this.dataType = udfParameters.getDataType(0); + valueReader = TypeServices.ROW_VALUE_READER_SERVICE.call(TypeServices.toReadType(dataType)); + numericReader = TypeServices.NUMERIC_ROW_READER_SERVICE.call(TypeServices.toReadType(dataType)); + valueWriter = TypeServices.rowValueWriter(dataType); this.windowSize = udfParameters.getIntOrDefault("window", 10000); this.v = new CircularQueue<>(windowSize); this.t = new LongCircularQueue(windowSize); @@ -73,33 +79,12 @@ public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConf @Override public void transform(Row row, PointCollector collector) throws Exception { - double value = Util.getValueAsDouble(row); + double value = numericReader.read(row); long timestamp = row.getTime(); if (Double.isFinite(value) && !Double.isNaN(value)) { if (v.isFull()) { final double frontValue = Double.parseDouble(v.pop().toString()); - switch (dataType) { - case INT32: - v.push(row.getInt(0)); - break; - case INT64: - v.push(row.getLong(0)); - break; - case DOUBLE: - v.push(row.getDouble(0)); - break; - case FLOAT: - v.push(row.getFloat(0)); - break; - case TIMESTAMP: - case DATE: - case TEXT: - case STRING: - case BLOB: - case BOOLEAN: - default: - break; - } + v.push(valueReader.read(row)); t.pop(); t.push(timestamp); this.sumX1 = this.sumX1 - frontValue + value; @@ -108,31 +93,10 @@ public void transform(Row row, PointCollector collector) throws Exception { this.variance = this.sumX2 / v.getSize() - this.mean * this.mean; if (Math.abs(value - mean) > multipleK * Math.sqrt(this.variance * v.getSize() / (v.getSize() - 1))) { - Util.putValue(collector, dataType, timestamp, Util.getValueAsObject(row)); + valueWriter.write(collector, timestamp, valueReader.read(row)); } } else { - switch (dataType) { - case INT32: - v.push(row.getInt(0)); - break; - case INT64: - v.push(row.getLong(0)); - break; - case DOUBLE: - v.push(row.getDouble(0)); - break; - case FLOAT: - v.push(row.getFloat(0)); - break; - case BLOB: - case BOOLEAN: - case STRING: - case TEXT: - case DATE: - case TIMESTAMP: - default: - break; - } + v.push(valueReader.read(row)); t.push(timestamp); this.sumX1 = this.sumX1 + value; this.sumX2 = this.sumX2 + value * value; @@ -144,7 +108,7 @@ public void transform(Row row, PointCollector collector) throws Exception { Object vi = this.v.get(i); timestamp = this.t.get(i); if (Math.abs(Double.parseDouble(vi.toString()) - mean) > multipleK * stddev) { - Util.putValue(collector, dataType, timestamp, vi); + valueWriter.write(collector, timestamp, vi); } } } @@ -160,7 +124,7 @@ public void terminate(PointCollector collector) throws Exception { Object vi = this.v.get(i); long timestamp = this.t.get(i); if (Math.abs(Double.parseDouble(vi.toString()) - mean) > multipleK * stddev) { - Util.putValue(collector, dataType, timestamp, vi); + valueWriter.write(collector, timestamp, vi); } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java index 783162aab690..f20d9737b1bd 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java @@ -19,7 +19,7 @@ package org.apache.iotdb.library.anomaly; -import org.apache.iotdb.library.i18n.LibraryUdfMessages; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; @@ -27,14 +27,14 @@ import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.RowByRowAccessStrategy; -import org.apache.iotdb.udf.api.exception.UDFException; import org.apache.iotdb.udf.api.type.Type; /** This function is used to detect range anomaly of time series. */ public class UDTFRange implements UDTF { - private Type dataType; private double upperBound; private double lowerBound; + private TypeServices.NumericRowReader rowReader; + private TypeServices.NumericRowWriter rowWriter; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -51,50 +51,17 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati .setOutputDataType(parameters.getDataType(0)); this.lowerBound = parameters.getDouble("lower_bound"); this.upperBound = parameters.getDouble("upper_bound"); - this.dataType = parameters.getDataType(0); + Type dataType = parameters.getDataType(0); + org.apache.tsfile.read.common.type.Type type = TypeServices.toReadType(dataType); + rowReader = TypeServices.NUMERIC_ROW_READER_SERVICE.call(type); + rowWriter = TypeServices.NUMERIC_ROW_WRITER_SERVICE.call(type); } @Override public void transform(Row row, PointCollector collector) throws Exception { - int intValue; - long longValue; - float floatValue; - double doubleValue; - long timestamp; - timestamp = row.getTime(); - switch (dataType) { - case INT32: - intValue = row.getInt(0); - if (intValue > upperBound || intValue < lowerBound) { - collector.putInt(timestamp, intValue); - } - break; - case INT64: - longValue = row.getLong(0); - if (longValue > upperBound || longValue < lowerBound) { - collector.putLong(timestamp, longValue); - } - break; - case FLOAT: - floatValue = row.getFloat(0); - if (floatValue > upperBound || floatValue < lowerBound) { - collector.putFloat(timestamp, floatValue); - } - break; - case DOUBLE: - doubleValue = row.getDouble(0); - if (doubleValue > upperBound || doubleValue < lowerBound) { - collector.putDouble(timestamp, doubleValue); - } - break; - case DATE: - case TIMESTAMP: - case TEXT: - case STRING: - case BOOLEAN: - case BLOB: - default: - throw new UDFException(LibraryUdfMessages.NO_SUCH_DATA_TYPE); + double value = rowReader.read(row); + if (value > upperBound || value < lowerBound) { + rowWriter.write(row, collector); } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFTwoSidedFilter.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFTwoSidedFilter.java index a772861f2d8a..f1cfe57a17f7 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFTwoSidedFilter.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFTwoSidedFilter.java @@ -20,7 +20,7 @@ package org.apache.iotdb.library.anomaly; import org.apache.iotdb.library.anomaly.util.WindowDetect; -import org.apache.iotdb.library.i18n.LibraryUdfMessages; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.RowWindow; import org.apache.iotdb.udf.api.collector.PointCollector; @@ -28,7 +28,6 @@ import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; -import org.apache.iotdb.udf.api.exception.UDFException; import org.apache.iotdb.udf.api.type.Type; /** @@ -38,6 +37,7 @@ public class UDTFTwoSidedFilter implements UDTF { private double len; private double threshold; + private TypeServices.NumericWindowWriter windowWriter; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -54,6 +54,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati .setOutputDataType(parameters.getDataType(0)); this.len = parameters.getDoubleOrDefault("len", 5); this.threshold = parameters.getDoubleOrDefault("threshold", 0.4); + windowWriter = + TypeServices.NUMERIC_WINDOW_WRITER_SERVICE.call( + TypeServices.toReadType(parameters.getDataType(0))); } @Override @@ -61,36 +64,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce WindowDetect wd = new WindowDetect(rowWindow.getRowIterator(), len, threshold); double[] repaired = wd.getRepaired(); long[] time = wd.getTime(); - switch (rowWindow.getDataType(0)) { - case DOUBLE: - for (int i = 0; i < time.length; i++) { - collector.putDouble(time[i], repaired[i]); - } - break; - case FLOAT: - for (int i = 0; i < time.length; i++) { - collector.putFloat(time[i], (float) repaired[i]); - } - break; - case INT32: - for (int i = 0; i < time.length; i++) { - collector.putInt(time[i], (int) Math.round(repaired[i])); - } - break; - case INT64: - for (int i = 0; i < time.length; i++) { - collector.putLong(time[i], Math.round(repaired[i])); - } - break; - case BOOLEAN: - case BLOB: - case STRING: - case TEXT: - case TIMESTAMP: - case DATE: - default: - throw new UDFException(LibraryUdfMessages.NO_SUCH_DATA_TYPE); - } + windowWriter.write(time, repaired, collector); } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPercentile.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPercentile.java index d3baa6a7bf0d..cab15d5244d1 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPercentile.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPercentile.java @@ -21,7 +21,7 @@ import org.apache.iotdb.library.dprofile.util.ExactOrderStatistics; import org.apache.iotdb.library.dprofile.util.GKArray; -import org.apache.iotdb.library.util.Util; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; @@ -44,7 +44,8 @@ public class UDAFPercentile implements UDTF { private GKArray sketch; private boolean exact; private double rank; - private Type dataType; + private TypeServices.NumericRowReader rowReader; + private PercentileOperations operations; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -67,7 +68,17 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati configurations .setAccessStrategy(new RowByRowAccessStrategy()) .setOutputDataType(parameters.getDataType(0)); - dataType = parameters.getDataType(0); + Type dataType = parameters.getDataType(0); + org.apache.tsfile.read.common.type.Type type = TypeServices.toReadType(dataType); + rowReader = TypeServices.NUMERIC_ROW_READER_SERVICE.call(type); + operations = + TypeServices.numericService( + INT_OPERATIONS, + LONG_OPERATIONS, + FLOAT_OPERATIONS, + DOUBLE_OPERATIONS, + UNSUPPORTED_OPERATIONS) + .call(type); double error = parameters.getDoubleOrDefault("error", 0); rank = parameters.getDoubleOrDefault("rank", 0.5); exact = (error == 0); @@ -76,120 +87,123 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati } else { sketch = new GKArray(error); } - switch (dataType) { - case INT32: - intDic = new HashMap<>(); - break; - case INT64: - longDic = new HashMap<>(); - break; - case FLOAT: - floatDic = new HashMap<>(); - break; - case DOUBLE: - doubleDic = new HashMap<>(); - break; - case TIMESTAMP: - case DATE: - case TEXT: - case STRING: - case BLOB: - case BOOLEAN: - default: - break; - } + operations.initialize(); } @Override public void transform(Row row, PointCollector collector) throws Exception { if (exact) { statistics.insert(row); - switch (dataType) { - case INT32: - intDic.put(row.getInt(0), row.getTime()); - break; - case INT64: - longDic.put(row.getLong(0), row.getTime()); - break; - case FLOAT: - floatDic.put(row.getFloat(0), row.getTime()); - break; - case DOUBLE: - doubleDic.put(row.getDouble(0), row.getTime()); - break; - case BLOB: - case BOOLEAN: - case STRING: - case TEXT: - case DATE: - case TIMESTAMP: - default: - break; - } + operations.insertExact(row); } else { - double res = Util.getValueAsDouble(row); - sketch.insert(res); + sketch.insert(rowReader.read(row)); } } @Override public void terminate(PointCollector collector) throws Exception { if (exact) { - long time; - switch (dataType) { - case INT32: - int ires = Integer.parseInt(statistics.getPercentile(rank)); - time = intDic.getOrDefault(ires, 0L); - collector.putInt(time, ires); - break; - case INT64: - long lres = Long.parseLong(statistics.getPercentile(rank)); - time = longDic.getOrDefault(lres, 0L); - collector.putLong(time, lres); - break; - case FLOAT: - float fres = Float.parseFloat(statistics.getPercentile(rank)); - time = floatDic.getOrDefault(fres, 0L); - collector.putFloat(time, fres); - break; - case DOUBLE: - double dres = Double.parseDouble(statistics.getPercentile(rank)); - time = doubleDic.getOrDefault(dres, 0L); - collector.putDouble(time, dres); - break; - case DATE: - case TIMESTAMP: - case TEXT: - case STRING: - case BOOLEAN: - case BLOB: - default: - break; - } + operations.writeExact(this, collector); } else { double res = sketch.query(rank); - switch (dataType) { - case INT32: - collector.putInt(0, (int) res); - break; - case INT64: - collector.putLong(0, (long) res); - break; - case FLOAT: - collector.putFloat(0, (float) res); - break; - case DOUBLE: - collector.putDouble(0, res); - break; - case BOOLEAN: - case BLOB: - case STRING: - case TEXT: - case TIMESTAMP: - case DATE: - default: - break; - } + operations.writeApprox(res, collector); } } + + private static final PercentileOperations INT_OPERATIONS = + new PercentileOperations() { + public void initialize() { + intDic = new HashMap<>(); + } + + public void insertExact(Row row) throws Exception { + intDic.put(row.getInt(0), row.getTime()); + } + + public void writeExact(UDAFPercentile target, PointCollector collector) throws Exception { + int value = Integer.parseInt(target.statistics.getPercentile(target.rank)); + collector.putInt(intDic.getOrDefault(value, 0L), value); + } + + public void writeApprox(double value, PointCollector collector) throws Exception { + collector.putInt(0, (int) value); + } + }; + private static final PercentileOperations LONG_OPERATIONS = + new PercentileOperations() { + public void initialize() { + longDic = new HashMap<>(); + } + + public void insertExact(Row row) throws Exception { + longDic.put(row.getLong(0), row.getTime()); + } + + public void writeExact(UDAFPercentile target, PointCollector collector) throws Exception { + long value = Long.parseLong(target.statistics.getPercentile(target.rank)); + collector.putLong(longDic.getOrDefault(value, 0L), value); + } + + public void writeApprox(double value, PointCollector collector) throws Exception { + collector.putLong(0, (long) value); + } + }; + private static final PercentileOperations FLOAT_OPERATIONS = + new PercentileOperations() { + public void initialize() { + floatDic = new HashMap<>(); + } + + public void insertExact(Row row) throws Exception { + floatDic.put(row.getFloat(0), row.getTime()); + } + + public void writeExact(UDAFPercentile target, PointCollector collector) throws Exception { + float value = Float.parseFloat(target.statistics.getPercentile(target.rank)); + collector.putFloat(floatDic.getOrDefault(value, 0L), value); + } + + public void writeApprox(double value, PointCollector collector) throws Exception { + collector.putFloat(0, (float) value); + } + }; + private static final PercentileOperations DOUBLE_OPERATIONS = + new PercentileOperations() { + public void initialize() { + doubleDic = new HashMap<>(); + } + + public void insertExact(Row row) throws Exception { + doubleDic.put(row.getDouble(0), row.getTime()); + } + + public void writeExact(UDAFPercentile target, PointCollector collector) throws Exception { + double value = Double.parseDouble(target.statistics.getPercentile(target.rank)); + collector.putDouble(doubleDic.getOrDefault(value, 0L), value); + } + + public void writeApprox(double value, PointCollector collector) throws Exception { + collector.putDouble(0, value); + } + }; + private static final PercentileOperations UNSUPPORTED_OPERATIONS = + new PercentileOperations() { + public void initialize() {} + + public void insertExact(Row row) {} + + public void writeExact(UDAFPercentile target, PointCollector collector) {} + + public void writeApprox(double value, PointCollector collector) {} + }; + + private interface PercentileOperations { + void initialize(); + + void insertExact(Row row) throws Exception; + + void writeExact(UDAFPercentile target, PointCollector collector) throws Exception; + + void writeApprox(double value, PointCollector collector) throws Exception; + } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java index 0518e7a3d768..4f8cbfc57efb 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java @@ -19,7 +19,8 @@ package org.apache.iotdb.library.dprofile; -import org.apache.iotdb.library.util.Util; +import org.apache.iotdb.library.i18n.LibraryUdfMessages; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; @@ -29,11 +30,69 @@ import org.apache.iotdb.udf.api.customizer.strategy.RowByRowAccessStrategy; import org.apache.iotdb.udf.api.type.Type; +import java.io.IOException; + /** calculate the approximate percentile. */ public class UDAFQuantile implements UDTF { private org.apache.iotdb.library.dprofile.util.HeapLongKLLSketch sketch; private double rank; - private Type dataType; + private QuantileOperations operations; + + private static final QuantileOperations INT_OPERATIONS = + new QuantileOperations() { + public long encode(Row row) throws IOException { + return row.getInt(0); + } + + public void write(long result, PointCollector collector) throws IOException { + collector.putInt(0, (int) result); + } + }; + private static final QuantileOperations LONG_OPERATIONS = + new QuantileOperations() { + public long encode(Row row) throws IOException { + return row.getLong(0); + } + + public void write(long result, PointCollector collector) throws IOException { + collector.putLong(0, result); + } + }; + private static final QuantileOperations FLOAT_OPERATIONS = + new QuantileOperations() { + public long encode(Row row) throws IOException { + float value = row.getFloat(0); + long bits = Float.floatToIntBits(value); + return value >= 0f ? bits : bits ^ Long.MAX_VALUE; + } + + public void write(long result, PointCollector collector) throws IOException { + result = (result >>> 31) == 0 ? result : result ^ Long.MAX_VALUE; + collector.putFloat(0, Float.intBitsToFloat((int) result)); + } + }; + private static final QuantileOperations DOUBLE_OPERATIONS = + new QuantileOperations() { + public long encode(Row row) throws IOException { + double value = row.getDouble(0); + long bits = Double.doubleToLongBits(value); + return value >= 0d ? bits : bits ^ Long.MAX_VALUE; + } + + public void write(long result, PointCollector collector) throws IOException { + result = (result >>> 63) == 0 ? result : result ^ Long.MAX_VALUE; + collector.putDouble(0, Double.longBitsToDouble(result)); + } + }; + private static final QuantileOperations UNSUPPORTED_OPERATIONS = + new QuantileOperations() { + public long encode(Row row) { + throw new IllegalArgumentException( + LibraryUdfMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_A8CA7BE7); + } + + public void write(long result, PointCollector collector) {} + }; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -42,11 +101,12 @@ public void validate(UDFParameterValidator validator) throws Exception { .validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE) .validate( k -> (int) k >= 100, - "Size K has to be greater or equal than 100.", + LibraryUdfMessages.EXCEPTION_SIZE_K_HAS_TO_BE_GREATER_THAN_OR_EQUAL_TO_100_C514D1C3, validator.getParameters().getIntOrDefault("K", 800)) .validate( rank -> (double) rank > 0 && (double) rank <= 1, - "rank has to be greater than 0 and less than or equal to 1.", + LibraryUdfMessages + .EXCEPTION_RANK_HAS_TO_BE_GREATER_THAN_0_AND_LESS_THAN_OR_EQUAL_TO_1_0F16AF94, validator.getParameters().getDoubleOrDefault("rank", 0.5)); } @@ -56,7 +116,15 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati configurations .setAccessStrategy(new RowByRowAccessStrategy()) .setOutputDataType(parameters.getDataType(0)); - dataType = parameters.getDataType(0); + Type dataType = parameters.getDataType(0); + operations = + TypeServices.numericService( + INT_OPERATIONS, + LONG_OPERATIONS, + FLOAT_OPERATIONS, + DOUBLE_OPERATIONS, + UNSUPPORTED_OPERATIONS) + .call(TypeServices.toReadType(dataType)); int k = parameters.getIntOrDefault("K", 800); rank = parameters.getDoubleOrDefault("rank", 0.5); @@ -65,19 +133,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { - final long encoded; - switch (dataType) { - case INT32: - encoded = row.getInt(0); - break; - case INT64: - encoded = row.getLong(0); - break; - default: - encoded = dataToLong(Util.getValueAsDouble(row)); - break; - } - sketch.update(encoded); + sketch.update(operations.encode(row)); } @Override @@ -94,49 +150,12 @@ public void terminate(PointCollector collector) throws Exception { k = n - 1; } } - long result = sketch.findMinValueWithRank(k); - switch (dataType) { - case INT32: - collector.putInt(0, (int) result); - break; - case INT64: - collector.putLong(0, result); - break; - case FLOAT: - collector.putFloat(0, (float) longToResult(result)); - break; - case DOUBLE: - collector.putDouble(0, longToResult(result)); - break; - default: - break; - } + operations.write(sketch.findMinValueWithRank(k), collector); } - private long dataToLong(double res) { - switch (dataType) { - case FLOAT: - float f = (float) res; - long flBits = Float.floatToIntBits(f); - return f >= 0f ? flBits : flBits ^ Long.MAX_VALUE; - case DOUBLE: - long d = Double.doubleToLongBits(res); - return res >= 0d ? d : d ^ Long.MAX_VALUE; - default: - return (long) res; - } - } + private interface QuantileOperations { + long encode(Row row) throws IOException; - private double longToResult(long result) { - switch (dataType) { - case FLOAT: - result = (result >>> 31) == 0 ? result : result ^ Long.MAX_VALUE; - return Float.intBitsToFloat((int) (result)); - case DOUBLE: - result = (result >>> 63) == 0 ? result : result ^ Long.MAX_VALUE; - return Double.longBitsToDouble(result); - default: - return (double) result; - } + void write(long result, PointCollector collector) throws IOException; } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSpread.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSpread.java index bc7d9ad7c846..5c947a846d1d 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSpread.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSpread.java @@ -20,6 +20,7 @@ package org.apache.iotdb.library.dprofile; import org.apache.iotdb.library.util.NoNumberException; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; @@ -45,7 +46,27 @@ public class UDAFSpread implements UDTF { float floatMax = -Float.MAX_VALUE; double doubleMin = Double.MAX_VALUE; double doubleMax = -Double.MAX_VALUE; - Type dataType; + private SpreadTransformer transformer; + private SpreadTerminator terminator; + + private static final SpreadTransformer INT_TRANSFORMER = UDAFSpread::transformInt; + private static final SpreadTransformer LONG_TRANSFORMER = UDAFSpread::transformLong; + private static final SpreadTransformer FLOAT_TRANSFORMER = UDAFSpread::transformFloat; + private static final SpreadTransformer DOUBLE_TRANSFORMER = UDAFSpread::transformDouble; + private static final SpreadTransformer UNSUPPORTED_TRANSFORMER = (target, row) -> {}; + + private static final SpreadTerminator INT_TERMINATOR = + (target, collector) -> collector.putInt(0, target.intMax - target.intMin); + private static final SpreadTerminator LONG_TERMINATOR = + (target, collector) -> collector.putLong(0, target.longMax - target.longMin); + private static final SpreadTerminator FLOAT_TERMINATOR = + (target, collector) -> collector.putFloat(0, target.floatMax - target.floatMin); + private static final SpreadTerminator DOUBLE_TERMINATOR = + (target, collector) -> collector.putDouble(0, target.doubleMax - target.doubleMin); + private static final SpreadTerminator UNSUPPORTED_TERMINATOR = + (target, collector) -> { + throw new NoNumberException(); + }; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -57,60 +78,35 @@ public void validate(UDFParameterValidator validator) throws Exception { @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { - dataType = parameters.getDataType(0); + Type dataType = parameters.getDataType(0); configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(dataType); + org.apache.tsfile.read.common.type.Type type = TypeServices.toReadType(dataType); + transformer = + TypeServices.numericService( + INT_TRANSFORMER, + LONG_TRANSFORMER, + FLOAT_TRANSFORMER, + DOUBLE_TRANSFORMER, + UNSUPPORTED_TRANSFORMER) + .call(type); + terminator = + TypeServices.numericService( + INT_TERMINATOR, + LONG_TERMINATOR, + FLOAT_TERMINATOR, + DOUBLE_TERMINATOR, + UNSUPPORTED_TERMINATOR) + .call(type); } @Override public void transform(Row row, PointCollector pc) throws Exception { - switch (dataType) { - case INT32: - transformInt(row); - break; - case INT64: - transformLong(row); - break; - case FLOAT: - transformFloat(row); - break; - case DOUBLE: - transformDouble(row); - break; - case BLOB: - case TIMESTAMP: - case BOOLEAN: - case STRING: - case TEXT: - case DATE: - default: - break; - } + transformer.transform(this, row); } @Override public void terminate(PointCollector pc) throws Exception { - switch (dataType) { - case INT32: - pc.putInt(0, intMax - intMin); - break; - case INT64: - pc.putLong(0, longMax - longMin); - break; - case FLOAT: - pc.putFloat(0, floatMax - floatMin); - break; - case DOUBLE: - pc.putDouble(0, doubleMax - doubleMin); - break; - case TEXT: - case DATE: - case STRING: - case BOOLEAN: - case TIMESTAMP: - case BLOB: - default: - throw new NoNumberException(); - } + terminator.terminate(this, pc); } private void transformInt(Row row) throws IOException { @@ -140,4 +136,15 @@ private void transformDouble(Row row) throws IOException { doubleMax = Math.max(doubleMax, v); } } + + @FunctionalInterface + private interface SpreadTransformer { + void transform(UDAFSpread target, Row row) throws IOException; + } + + @FunctionalInterface + private interface SpreadTerminator { + void terminate(UDAFSpread target, PointCollector collector) + throws IOException, NoNumberException; + } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFDistinct.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFDistinct.java index 1698164a944f..55e4deff22b8 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFDistinct.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFDistinct.java @@ -19,6 +19,7 @@ package org.apache.iotdb.library.dprofile; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; @@ -50,7 +51,7 @@ public class UDTFDistinct implements UDTF { private DoubleHashSet doubleSet; private BooleanHashSet booleanSet; private HashSet stringSet; - private Type dataType; + private DistinctOperations operations; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -66,146 +67,184 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati configurations .setAccessStrategy(new RowByRowAccessStrategy()) .setOutputDataType(parameters.getDataType(0)); - dataType = parameters.getDataType(0); - switch (dataType) { - case INT32: - intSet = new IntHashSet(); - break; - case INT64: - longSet = new LongHashSet(); - break; - case FLOAT: - floatSet = new FloatHashSet(); - break; - case DOUBLE: - doubleSet = new DoubleHashSet(); - break; - case TEXT: - stringSet = new HashSet<>(); - break; - case BOOLEAN: - booleanSet = new BooleanHashSet(); - break; - case BLOB: - case STRING: - case TIMESTAMP: - case DATE: - default: - break; - } + Type dataType = parameters.getDataType(0); + operations = + TypeServices.scalarService( + BOOLEAN_OPERATIONS, + INT_OPERATIONS, + LONG_OPERATIONS, + FLOAT_OPERATIONS, + DOUBLE_OPERATIONS, + TEXT_OPERATIONS, + UNSUPPORTED_OPERATIONS) + .call(TypeServices.toReadType(dataType)); + operations.initialize(this); } @Override public void transform(Row row, PointCollector pc) throws Exception { - switch (dataType) { - case INT32: - intSet.add(row.getInt(0)); - break; - case INT64: - longSet.add(row.getLong(0)); - break; - case FLOAT: - floatSet.add(row.getFloat(0)); - break; - case DOUBLE: - doubleSet.add(row.getDouble(0)); - break; - case TEXT: - stringSet.add(row.getString(0)); - break; - case BOOLEAN: - booleanSet.add(row.getBoolean(0)); - break; - case BLOB: - case STRING: - case TIMESTAMP: - case DATE: - default: - break; - } + operations.add(this, row); } @Override public void terminate(PointCollector pc) throws Exception { - int i = 0; - switch (dataType) { - case INT32: - MutableIntIterator intIterator = intSet.intIterator(); - while (intIterator.hasNext()) { - pc.putInt(i, intIterator.next()); - i++; - } - break; - case INT64: - MutableLongIterator longIterator = longSet.longIterator(); - while (longIterator.hasNext()) { - pc.putLong(i, longIterator.next()); - i++; - } - break; - case FLOAT: - MutableFloatIterator floatIterator = floatSet.floatIterator(); - while (floatIterator.hasNext()) { - pc.putFloat(i, floatIterator.next()); - i++; - } - break; - case DOUBLE: - MutableDoubleIterator doubleIterator = doubleSet.doubleIterator(); - while (doubleIterator.hasNext()) { - pc.putDouble(i, doubleIterator.next()); - i++; - } - break; - case TEXT: - for (String s : stringSet) { - pc.putString(i, s); - i++; - } - break; - case BOOLEAN: - MutableBooleanIterator booleanIterator = booleanSet.booleanIterator(); - while (booleanIterator.hasNext()) { - pc.putBoolean(i, booleanIterator.next()); - i++; - } - break; - case BLOB: - case STRING: - case TIMESTAMP: - case DATE: - default: - break; - } + operations.terminate(this, pc); } @Override public void beforeDestroy() { - switch (dataType) { - case INT32: - intSet.clear(); - break; - case INT64: - longSet.clear(); - break; - case FLOAT: - floatSet.clear(); - break; - case DOUBLE: - doubleSet.clear(); - break; - case TEXT: - stringSet.clear(); - break; - case BOOLEAN: - booleanSet.clear(); - break; - case BLOB: - case STRING: - case TIMESTAMP: - case DATE: - default: - break; - } + operations.clear(this); + } + + private static final DistinctOperations INT_OPERATIONS = + new DistinctOperations() { + public void initialize(UDTFDistinct target) { + target.intSet = new IntHashSet(); + } + + public void add(UDTFDistinct target, Row row) throws Exception { + target.intSet.add(row.getInt(0)); + } + + public void terminate(UDTFDistinct target, PointCollector collector) throws Exception { + MutableIntIterator iterator = target.intSet.intIterator(); + int i = 0; + while (iterator.hasNext()) { + collector.putInt(i++, iterator.next()); + } + } + + public void clear(UDTFDistinct target) { + target.intSet.clear(); + } + }; + private static final DistinctOperations LONG_OPERATIONS = + new DistinctOperations() { + public void initialize(UDTFDistinct target) { + target.longSet = new LongHashSet(); + } + + public void add(UDTFDistinct target, Row row) throws Exception { + target.longSet.add(row.getLong(0)); + } + + public void terminate(UDTFDistinct target, PointCollector collector) throws Exception { + MutableLongIterator iterator = target.longSet.longIterator(); + int i = 0; + while (iterator.hasNext()) { + collector.putLong(i++, iterator.next()); + } + } + + public void clear(UDTFDistinct target) { + target.longSet.clear(); + } + }; + private static final DistinctOperations FLOAT_OPERATIONS = + new DistinctOperations() { + public void initialize(UDTFDistinct target) { + target.floatSet = new FloatHashSet(); + } + + public void add(UDTFDistinct target, Row row) throws Exception { + target.floatSet.add(row.getFloat(0)); + } + + public void terminate(UDTFDistinct target, PointCollector collector) throws Exception { + MutableFloatIterator iterator = target.floatSet.floatIterator(); + int i = 0; + while (iterator.hasNext()) { + collector.putFloat(i++, iterator.next()); + } + } + + public void clear(UDTFDistinct target) { + target.floatSet.clear(); + } + }; + private static final DistinctOperations DOUBLE_OPERATIONS = + new DistinctOperations() { + public void initialize(UDTFDistinct target) { + target.doubleSet = new DoubleHashSet(); + } + + public void add(UDTFDistinct target, Row row) throws Exception { + target.doubleSet.add(row.getDouble(0)); + } + + public void terminate(UDTFDistinct target, PointCollector collector) throws Exception { + MutableDoubleIterator iterator = target.doubleSet.doubleIterator(); + int i = 0; + while (iterator.hasNext()) { + collector.putDouble(i++, iterator.next()); + } + } + + public void clear(UDTFDistinct target) { + target.doubleSet.clear(); + } + }; + private static final DistinctOperations BOOLEAN_OPERATIONS = + new DistinctOperations() { + public void initialize(UDTFDistinct target) { + target.booleanSet = new BooleanHashSet(); + } + + public void add(UDTFDistinct target, Row row) throws Exception { + target.booleanSet.add(row.getBoolean(0)); + } + + public void terminate(UDTFDistinct target, PointCollector collector) throws Exception { + MutableBooleanIterator iterator = target.booleanSet.booleanIterator(); + int i = 0; + while (iterator.hasNext()) { + collector.putBoolean(i++, iterator.next()); + } + } + + public void clear(UDTFDistinct target) { + target.booleanSet.clear(); + } + }; + private static final DistinctOperations TEXT_OPERATIONS = + new DistinctOperations() { + public void initialize(UDTFDistinct target) { + target.stringSet = new HashSet<>(); + } + + public void add(UDTFDistinct target, Row row) throws Exception { + target.stringSet.add(row.getString(0)); + } + + public void terminate(UDTFDistinct target, PointCollector collector) throws Exception { + int i = 0; + for (String value : target.stringSet) { + collector.putString(i++, value); + } + } + + public void clear(UDTFDistinct target) { + target.stringSet.clear(); + } + }; + private static final DistinctOperations UNSUPPORTED_OPERATIONS = + new DistinctOperations() { + public void initialize(UDTFDistinct target) {} + + public void add(UDTFDistinct target, Row row) {} + + public void terminate(UDTFDistinct target, PointCollector collector) {} + + public void clear(UDTFDistinct target) {} + }; + + private interface DistinctOperations { + void initialize(UDTFDistinct target) throws Exception; + + void add(UDTFDistinct target, Row row) throws Exception; + + void terminate(UDTFDistinct target, PointCollector collector) throws Exception; + + void clear(UDTFDistinct target); } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSample.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSample.java index 545d4096f51b..27918e5238dd 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSample.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSample.java @@ -20,8 +20,7 @@ package org.apache.iotdb.library.dprofile; import org.apache.iotdb.library.dprofile.util.LTThreeBuckets; -import org.apache.iotdb.library.util.NoNumberException; -import org.apache.iotdb.library.util.Util; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.access.RowIterator; @@ -58,6 +57,10 @@ enum Method { private int num = 0; // number of points already sampled private Random random; private Type dataType; + private TypeServices.NumericValueWriter numericValueWriter; + private TypeServices.NumericRowReader numericRowReader; + private TypeServices.RowValueReader valueReader; + private TypeServices.RowValueWriter valueWriter; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -81,6 +84,12 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati throws Exception { this.k = parameters.getIntOrDefault("k", 1); this.dataType = parameters.getDataType(0); + numericValueWriter = + TypeServices.NUMERIC_VALUE_WRITER_SERVICE.call(TypeServices.toReadType(dataType)); + numericRowReader = + TypeServices.NUMERIC_ROW_READER_SERVICE.call(TypeServices.toReadType(dataType)); + valueReader = TypeServices.rowValueReader(dataType); + valueWriter = TypeServices.rowValueWriter(dataType); String methodIn = parameters.getStringOrDefault("method", METHOD_RESERVOIR); if ("triangle".equalsIgnoreCase(methodIn)) { this.method = Method.TRIANGLE; @@ -112,7 +121,7 @@ public void transform(Row row, PointCollector collector) throws Exception { x = random.nextInt(num + 1); } if (x < this.k) { - Object v = Util.getValueAsObject(row); + Object v = valueReader.read(row); Long t = row.getTime(); this.samples[x] = Pair.of(t, v); } @@ -131,56 +140,35 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce for (int i = 0; i < n; i++) { Row row = rowWindow.getRow(i); long time = row.getTime(); - double data = Util.getValueAsDouble(row); + double data = numericRowReader.read(row); input.add(Pair.of(time, data)); } if (k > 2) { // The first and last element will always be sampled so the buckets is k - 2 List> output = LTThreeBuckets.sorted(input, k - 2); for (Pair p : output) { - switch (dataType) { - case INT32: - collector.putInt(p.getLeft(), p.getRight().intValue()); - break; - case INT64: - collector.putLong(p.getLeft(), p.getRight().longValue()); - break; - case FLOAT: - collector.putFloat(p.getLeft(), p.getRight().floatValue()); - break; - case DOUBLE: - collector.putDouble(p.getLeft(), p.getRight()); - break; - case TIMESTAMP: - case DATE: - case BLOB: - case BOOLEAN: - case STRING: - case TEXT: - default: - throw new NoNumberException(); - } + numericValueWriter.write(p.getLeft(), p.getRight(), collector); } } else { // For corner case of k == 1 and k == 2 Row row = rowWindow.getRow(0); // Put first element - Util.putValue(collector, dataType, row.getTime(), Util.getValueAsObject(row)); + valueWriter.write(collector, row.getTime(), valueReader.read(row)); if (k == 2) { row = rowWindow.getRow(n - 1); // Put last element - Util.putValue(collector, dataType, row.getTime(), Util.getValueAsObject(row)); + valueWriter.write(collector, row.getTime(), valueReader.read(row)); } } } else { // Method.ISOMETRIC for (long i = 0; i < this.k; i++) { long j = Math.floorDiv(i * n, (long) k); // avoid intermediate result overflows Row row = rowWindow.getRow((int) j); - Util.putValue(collector, dataType, row.getTime(), Util.getValueAsObject(row)); + valueWriter.write(collector, row.getTime(), valueReader.read(row)); } } } else { // when k is larger than series length, output all points RowIterator iterator = rowWindow.getRowIterator(); while (iterator.hasNextRow()) { Row row = iterator.next(); - Util.putValue(collector, dataType, row.getTime(), Util.getValueAsObject(row)); + valueWriter.write(collector, row.getTime(), valueReader.read(row)); } } } @@ -192,7 +180,7 @@ public void terminate(PointCollector pc) throws Exception { Arrays.sort(samples, 0, m); for (int i = 0; i < m; i++) { Pair p = samples[i]; - Util.putValue(pc, dataType, p.getLeft(), p.getRight()); + valueWriter.write(pc, p.getLeft(), p.getRight()); } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java index 47ca5e2b12fd..c4c49f0fdb9f 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java @@ -19,10 +19,13 @@ package org.apache.iotdb.library.dprofile.util; +import org.apache.iotdb.library.i18n.LibraryUdfMessages; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; import org.apache.iotdb.udf.api.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList; import org.eclipse.collections.impl.list.mutable.primitive.FloatArrayList; import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; @@ -30,6 +33,7 @@ import java.io.IOException; import java.util.NoSuchElementException; +import java.util.function.Function; /** * Util for computing median, MAD, percentile. @@ -46,71 +50,24 @@ public class ExactOrderStatistics { private DoubleArrayList doubleArrayList; private IntArrayList intArrayList; private LongArrayList longArrayList; + private final StatisticsOperations operations; public ExactOrderStatistics(Type type) throws UDFInputSeriesDataTypeNotValidException { this.dataType = type; - switch (dataType) { - case INT32: - intArrayList = new IntArrayList(); - break; - case INT64: - longArrayList = new LongArrayList(); - break; - case FLOAT: - floatArrayList = new FloatArrayList(); - break; - case DOUBLE: - doubleArrayList = new DoubleArrayList(); - break; - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, dataType, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); + try { + operations = OPERATIONS_SERVICE.call(TypeServices.toReadType(type)).apply(this); + } catch (IllegalArgumentException e) { + throw new UDFInputSeriesDataTypeNotValidException( + 0, dataType, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); } } public void insert(Row row) throws UDFInputSeriesDataTypeNotValidException, IOException { - switch (dataType) { - case INT32: - intArrayList.add(row.getInt(0)); - break; - case INT64: - longArrayList.add(row.getLong(0)); - break; - case FLOAT: - float vf = row.getFloat(0); - if (Float.isFinite(vf)) { - floatArrayList.add(vf); - } - break; - case DOUBLE: - double vd = row.getDouble(0); - if (Double.isFinite(vd)) { - doubleArrayList.add(vd); - } - break; - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, dataType, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); - } + operations.insert(row); } public double getMedian() throws UDFInputSeriesDataTypeNotValidException { - switch (dataType) { - case INT32: - return getMedian(intArrayList); - case INT64: - return getMedian(longArrayList); - case FLOAT: - return getMedian(floatArrayList); - case DOUBLE: - return getMedian(doubleArrayList); - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, dataType, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); - } + return operations.median(); } public static double getMedian(FloatArrayList nums) { @@ -179,20 +136,7 @@ public static double getMad(FloatArrayList nums) { } public double getMad() throws UDFInputSeriesDataTypeNotValidException { - switch (dataType) { - case INT32: - return getMad(intArrayList); - case INT64: - return getMad(longArrayList); - case FLOAT: - return getMad(floatArrayList); - case DOUBLE: - return getMad(doubleArrayList); - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, dataType, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); - } + return operations.mad(); } public static double getMad(DoubleArrayList nums) { @@ -259,20 +203,120 @@ public static double getPercentile(DoubleArrayList nums, double phi) { } public String getPercentile(double phi) throws UDFInputSeriesDataTypeNotValidException { - switch (dataType) { - case INT32: - return Integer.toString(getPercentile(intArrayList, phi)); - case INT64: - return Long.toString(getPercentile(longArrayList, phi)); - case FLOAT: - return Float.toString(getPercentile(floatArrayList, phi)); - case DOUBLE: - return Double.toString(getPercentile(doubleArrayList, phi)); - default: - // This will not happen. - throw new UDFInputSeriesDataTypeNotValidException( - 0, dataType, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); - } + return operations.percentile(phi); + } + + private static final TypeService> + OPERATIONS_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + target -> { + target.intArrayList = new IntArrayList(); + return new StatisticsOperations() { + public void insert(Row row) throws IOException { + target.intArrayList.add(row.getInt(0)); + } + + public double median() { + return getMedian(target.intArrayList); + } + + public double mad() { + return getMad(target.intArrayList); + } + + public String percentile(double phi) { + return Integer.toString(getPercentile(target.intArrayList, phi)); + } + }; + }; + case INT64 -> + target -> { + target.longArrayList = new LongArrayList(); + return new StatisticsOperations() { + public void insert(Row row) throws IOException { + target.longArrayList.add(row.getLong(0)); + } + + public double median() { + return getMedian(target.longArrayList); + } + + public double mad() { + return getMad(target.longArrayList); + } + + public String percentile(double phi) { + return Long.toString(getPercentile(target.longArrayList, phi)); + } + }; + }; + case FLOAT -> + target -> { + target.floatArrayList = new FloatArrayList(); + return new StatisticsOperations() { + public void insert(Row row) throws IOException { + float value = row.getFloat(0); + if (Float.isFinite(value)) { + target.floatArrayList.add(value); + } + } + + public double median() { + return getMedian(target.floatArrayList); + } + + public double mad() { + return getMad(target.floatArrayList); + } + + public String percentile(double phi) { + return Float.toString(getPercentile(target.floatArrayList, phi)); + } + }; + }; + case DOUBLE -> + target -> { + target.doubleArrayList = new DoubleArrayList(); + return new StatisticsOperations() { + public void insert(Row row) throws IOException { + double value = row.getDouble(0); + if (Double.isFinite(value)) { + target.doubleArrayList.add(value); + } + } + + public double median() { + return getMedian(target.doubleArrayList); + } + + public double mad() { + return getMad(target.doubleArrayList); + } + + public String percentile(double phi) { + return Double.toString(getPercentile(target.doubleArrayList, phi)); + } + }; + }; + case BOOLEAN, TEXT, ROW, UNKNOWN, TIMESTAMP, DATE, BLOB, STRING, OBJECT, VECTOR -> + target -> { + throw new IllegalArgumentException( + String.format( + LibraryUdfMessages.EXCEPTION_UNSUPPORTED_DATA_TYPE_ARG_B411C29E, + type)); + }; + }; + + private interface StatisticsOperations { + void insert(Row row) throws IOException; + + double median(); + + double mad(); + + String percentile(double phi); } public static int getPercentile(IntArrayList nums, double phi) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFTimestampRepair.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFTimestampRepair.java index 873754a11943..9605d5e4d163 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFTimestampRepair.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFTimestampRepair.java @@ -21,6 +21,7 @@ import org.apache.iotdb.library.drepair.util.TimestampRepair; import org.apache.iotdb.library.i18n.LibraryUdfMessages; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.library.util.Util; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.RowWindow; @@ -37,6 +38,7 @@ public class UDTFTimestampRepair implements UDTF { String intervalMethod; long interval; long intervalMode; + private TypeServices.NumericWindowWriter windowWriter; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -67,6 +69,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati configurations .setAccessStrategy(new SlidingSizeWindowAccessStrategy(Integer.MAX_VALUE)) .setOutputDataType(parameters.getDataType(0)); + windowWriter = + TypeServices.NUMERIC_CAST_WINDOW_WRITER_SERVICE.call( + TypeServices.toReadType(parameters.getDataType(0))); intervalMethod = parameters.getStringOrDefault("method", "Median"); String intervalString = parameters.getStringOrDefault("interval", null); @@ -100,35 +105,6 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce ts.dpRepair(); long[] timestamp = ts.getRepaired(); double[] value = ts.getRepairedValue(); - switch (rowWindow.getDataType(0)) { - case DOUBLE: - for (int i = 0; i < timestamp.length; i++) { - collector.putDouble(timestamp[i], value[i]); - } - break; - case FLOAT: - for (int i = 0; i < timestamp.length; i++) { - collector.putFloat(timestamp[i], (float) value[i]); - } - break; - case INT32: - for (int i = 0; i < timestamp.length; i++) { - collector.putInt(timestamp[i], (int) value[i]); - } - break; - case INT64: - for (int i = 0; i < timestamp.length; i++) { - collector.putLong(timestamp[i], (long) value[i]); - } - break; - case DATE: - case TIMESTAMP: - case BLOB: - case BOOLEAN: - case TEXT: - case STRING: - default: - throw new UDFException(""); - } + windowWriter.write(timestamp, value, collector); } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueFill.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueFill.java index 0b6500153689..ed795ea0be16 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueFill.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueFill.java @@ -27,6 +27,7 @@ import org.apache.iotdb.library.drepair.util.ScreenFill; import org.apache.iotdb.library.drepair.util.ValueFill; import org.apache.iotdb.library.i18n.LibraryUdfMessages; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.RowWindow; import org.apache.iotdb.udf.api.collector.PointCollector; @@ -40,6 +41,7 @@ /** This function is used to interpolate time series. */ public class UDTFValueFill implements UDTF { private String method; + private TypeServices.NumericWindowWriter windowWriter; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -55,6 +57,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati .setAccessStrategy(new SlidingSizeWindowAccessStrategy(Integer.MAX_VALUE)) .setOutputDataType(parameters.getDataType(0)); method = parameters.getStringOrDefault("method", "linear"); + windowWriter = + TypeServices.NUMERIC_WINDOW_WRITER_SERVICE.call( + TypeServices.toReadType(parameters.getDataType(0))); } @Override @@ -78,35 +83,6 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce vf.fill(); double[] repaired = vf.getFilled(); long[] time = vf.getTime(); - switch (rowWindow.getDataType(0)) { - case DOUBLE: - for (int i = 0; i < time.length; i++) { - collector.putDouble(time[i], repaired[i]); - } - break; - case FLOAT: - for (int i = 0; i < time.length; i++) { - collector.putFloat(time[i], (float) repaired[i]); - } - break; - case INT32: - for (int i = 0; i < time.length; i++) { - collector.putInt(time[i], (int) Math.round(repaired[i])); - } - break; - case INT64: - for (int i = 0; i < time.length; i++) { - collector.putLong(time[i], Math.round(repaired[i])); - } - break; - case TEXT: - case STRING: - case BOOLEAN: - case BLOB: - case TIMESTAMP: - case DATE: - default: - throw new UDFException(""); - } + windowWriter.write(time, repaired, collector); } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueRepair.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueRepair.java index a4d9b18ef60a..7a0a1990d0f3 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueRepair.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueRepair.java @@ -23,6 +23,7 @@ import org.apache.iotdb.library.drepair.util.Screen; import org.apache.iotdb.library.drepair.util.ValueRepair; import org.apache.iotdb.library.i18n.LibraryUdfMessages; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.RowWindow; import org.apache.iotdb.udf.api.collector.PointCollector; @@ -40,6 +41,7 @@ public class UDTFValueRepair implements UDTF { double maxSpeed; double center; double sigma; + private TypeServices.NumericWindowWriter windowWriter; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -68,6 +70,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati maxSpeed = parameters.getDoubleOrDefault("maxSpeed", Double.NaN); center = parameters.getDoubleOrDefault("center", 0); sigma = parameters.getDoubleOrDefault("sigma", Double.NaN); + windowWriter = + TypeServices.NUMERIC_WINDOW_WRITER_SERVICE.call( + TypeServices.toReadType(parameters.getDataType(0))); } @Override @@ -95,36 +100,6 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce vr.repair(); double[] repaired = vr.getRepaired(); long[] time = vr.getTime(); - switch (rowWindow.getDataType(0)) { - case DOUBLE: - for (int i = 0; i < time.length; i++) { - collector.putDouble(time[i], repaired[i]); - } - break; - case FLOAT: - for (int i = 0; i < time.length; i++) { - collector.putFloat(time[i], (float) repaired[i]); - } - break; - case INT32: - for (int i = 0; i < time.length; i++) { - collector.putInt(time[i], (int) Math.round(repaired[i])); - } - break; - case INT64: - for (int i = 0; i < time.length; i++) { - collector.putLong(time[i], Math.round(repaired[i])); - } - break; - case TIMESTAMP: - case DATE: - case BLOB: - case BOOLEAN: - case STRING: - case TEXT: - - default: - throw new Exception(); - } + windowWriter.write(time, repaired, collector); } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java index 6bf3ba151bed..b0e3742c670e 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java @@ -20,6 +20,8 @@ package org.apache.iotdb.library.frequency; import org.apache.iotdb.library.i18n.LibraryUdfMessages; +import org.apache.iotdb.library.util.NoNumberException; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; @@ -46,6 +48,7 @@ public class UDFEnvelopeAnalysis implements UDTF { private String timestampPrecision; private final DoubleArrayList signals = new DoubleArrayList(); private final LongArrayList timestamps = new LongArrayList(); + private TypeServices.NumericRowReader valueReader; private static final String TIMESTAMP_PRECISION = "timestampPrecision"; private static final String FREQUENCY = "frequency"; private static final String AMPLIFICATION = "amplification"; @@ -60,11 +63,12 @@ public void validate(UDFParameterValidator validator) throws Exception { .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) .validate( x -> (double) x > 0, - "The param 'frequency' must > 0.", + LibraryUdfMessages.EXCEPTION_THE_PARAM_FREQUENCY_MUST_BE_GREATER_THAN_0_45820CF9, validator.getParameters().getDoubleOrDefault(FREQUENCY, Double.MAX_VALUE)) .validate( x -> (int) x >= 1, - "The param 'amplification' must >= 1.", + LibraryUdfMessages + .EXCEPTION_THE_PARAM_AMPLIFICATION_MUST_BE_GREATER_THAN_OR_EQUAL_TO_1_D64050EB, validator.getParameters().getIntOrDefault(AMPLIFICATION, 1)); } @@ -75,11 +79,14 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati frequency = parameters.getDoubleOrDefault(FREQUENCY, Double.MAX_VALUE); amplification = parameters.getIntOrDefault(AMPLIFICATION, 1); timestampPrecision = parameters.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); + valueReader = + TypeServices.NUMERIC_ROW_READER_SERVICE.call( + TypeServices.toReadType(parameters.getDataType(0))); } @Override public void transform(Row row, PointCollector collector) throws Exception { - signals.add(getValueAsDouble(row, 0)); + signals.add(valueReader.read(row)); if (timestamps.size() < 10) { timestamps.add(row.getTime()); } @@ -213,32 +220,14 @@ public long calculateMode(LongArrayList timestamps) { } public double getValueAsDouble(Row row, int index) throws IOException { - double ans; - switch (row.getDataType(index)) { - case INT32: - ans = row.getInt(index); - break; - case INT64: - ans = row.getLong(index); - break; - case FLOAT: - ans = row.getFloat(index); - break; - case DOUBLE: - ans = row.getDouble(index); - break; - case TEXT: - case STRING: - case BOOLEAN: - case BLOB: - case DATE: - case TIMESTAMP: - default: - throw new UDFOutputSeriesDataTypeNotValidException( - index, - LibraryUdfMessages.EXCEPTION_FAIL_TO_GET_DATA_TYPE_IN_ROW_8CD82629 + row.getTime()); + try { + return TypeServices.NUMERIC_ROW_READER_SERVICE + .call(TypeServices.toReadType(row.getDataType(index))) + .read(row); + } catch (NoNumberException e) { + throw new UDFOutputSeriesDataTypeNotValidException( + index, LibraryUdfMessages.FAIL_TO_GET_DATA_TYPE_IN_ROW + row.getTime()); } - return ans; } public static double calculateFrequencyByTimeUnit(long time, String timeUnit) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFDTWMatch.java b/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFDTWMatch.java index 47646fd5fe90..f5aa0418471a 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFDTWMatch.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFDTWMatch.java @@ -19,9 +19,9 @@ package org.apache.iotdb.library.match; -import org.apache.iotdb.library.i18n.LibraryUdfMessages; import org.apache.iotdb.library.match.model.DTWMatchResult; import org.apache.iotdb.library.match.model.DTWState; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.State; import org.apache.iotdb.udf.api.UDAF; import org.apache.iotdb.udf.api.customizer.config.UDAFConfigurations; @@ -44,12 +44,16 @@ public class UDAFDTWMatch implements UDAF { private Double[] pattern; private float threshold; private DTWState state; + private TypeServices.ColumnNumericReader valueReader; @Override public void beforeStart(UDFParameters udfParameters, UDAFConfigurations udafConfigurations) { udafConfigurations.setOutputDataType(Type.TEXT); Map attributes = udfParameters.getAttributes(); threshold = Float.parseFloat(attributes.get("threshold")); + valueReader = + TypeServices.COLUMN_NUMERIC_READER_SERVICE.call( + TypeServices.toReadType(udfParameters.getDataType(0))); pattern = Arrays.stream(attributes.get("pattern").split(",")) .map(Double::valueOf) @@ -81,7 +85,7 @@ public void addInput(State state, Column[] columns, BitMap bitMap) { } if (!columns[1].isNull(i)) { long timestamp = columns[1].getLong(i); - double value = getValue(columns[0], i); + double value = valueReader.read(columns[0], i); DTWState.updateBuffer(timestamp, value); if (DTWState.getValueBuffer().length == pattern.length) { float dtw = calculateDTW(DTWState.getValueBuffer(), pattern); @@ -95,24 +99,6 @@ public void addInput(State state, Column[] columns, BitMap bitMap) { } } - private double getValue(Column column, int i) { - switch (column.getDataType()) { - case INT32: - return column.getInt(i); - case INT64: - return column.getLong(i); - case FLOAT: - return column.getFloat(i); - case DOUBLE: - return column.getDouble(i); - case BOOLEAN: - return column.getBoolean(i) ? 1.0D : 0.0D; - default: - throw new RuntimeException( - String.format(LibraryUdfMessages.UNSUPPORTED_DATATYPE, column.getDataType())); - } - } - private float calculateDTW(Double[] series1, Double[] series2) { int n = series1.length; double[][] dtw = new double[n][n]; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFPatternMatch.java b/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFPatternMatch.java index cc814a3bdd81..5e6308af1f4e 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFPatternMatch.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFPatternMatch.java @@ -25,6 +25,7 @@ import org.apache.iotdb.library.match.model.PatternResult; import org.apache.iotdb.library.match.model.PatternState; import org.apache.iotdb.library.match.model.Point; +import org.apache.iotdb.library.util.TypeServices; import org.apache.iotdb.udf.api.State; import org.apache.iotdb.udf.api.UDAF; import org.apache.iotdb.udf.api.customizer.config.UDAFConfigurations; @@ -54,12 +55,16 @@ public class UDAFPatternMatch implements UDAF { private Double[] valuePattern; private float threshold; private PatternState state; + private TypeServices.ColumnNumericReader valueReader; @Override public void beforeStart(UDFParameters udfParameters, UDAFConfigurations udafConfigurations) { udafConfigurations.setOutputDataType(Type.TEXT); Map attributes = udfParameters.getAttributes(); threshold = Float.parseFloat(attributes.get(THRESHOLD_PARAM)); + valueReader = + TypeServices.COLUMN_NUMERIC_READER_SERVICE.call( + TypeServices.toReadType(udfParameters.getDataType(0))); } @Override @@ -79,7 +84,7 @@ public void addInput(State state, Column[] columns, BitMap bitMap) { } if (!columns[1].isNull(i)) { long timestamp = columns[1].getLong(i); - double value = getValue(columns[0], i); + double value = valueReader.read(columns[0], i); matchState.updateBuffer(timestamp, value); } } @@ -176,22 +181,4 @@ public void validate(UDFParameterValidator validator) { timePattern, valuePattern); } - - private double getValue(Column column, int i) { - switch (column.getDataType()) { - case INT32: - return column.getInt(i); - case INT64: - return column.getLong(i); - case FLOAT: - return column.getFloat(i); - case DOUBLE: - return column.getDouble(i); - case BOOLEAN: - return column.getBoolean(i) ? 1.0D : 0.0D; - default: - throw new RuntimeException( - String.format(LibraryUdfMessages.UNSUPPORTED_DATATYPE, column.getDataType())); - } - } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/util/TypeServices.java b/library-udf/src/main/java/org/apache/iotdb/library/util/TypeServices.java new file mode 100644 index 000000000000..9081936eba45 --- /dev/null +++ b/library-udf/src/main/java/org/apache/iotdb/library/util/TypeServices.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.library.util; + +import org.apache.iotdb.udf.api.access.Row; +import org.apache.iotdb.udf.api.collector.PointCollector; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.service.TypeService; + +import java.io.IOException; + +/** Type-specific operations shared by library UDFs. */ +public final class TypeServices { + + public static final TypeService NUMERIC_ROW_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> row -> row.getInt(0); + case INT64 -> row -> row.getLong(0); + case FLOAT -> row -> row.getFloat(0); + case DOUBLE -> row -> row.getDouble(0); + default -> + row -> { + throw new NoNumberException(); + }; + }; + + public static final TypeService INDEXED_NUMERIC_ROW_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> Row::getInt; + case INT64 -> Row::getLong; + case FLOAT -> Row::getFloat; + case DOUBLE -> Row::getDouble; + default -> + (row, index) -> { + throw new NoNumberException(); + }; + }; + + public static final TypeService NUMERIC_ROW_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> (row, collector) -> collector.putInt(row.getTime(), row.getInt(0)); + case INT64 -> (row, collector) -> collector.putLong(row.getTime(), row.getLong(0)); + case FLOAT -> (row, collector) -> collector.putFloat(row.getTime(), row.getFloat(0)); + case DOUBLE -> (row, collector) -> collector.putDouble(row.getTime(), row.getDouble(0)); + default -> + (row, collector) -> { + throw new NoNumberException(); + }; + }; + + public static final TypeService ROW_VALUE_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> row -> row.getInt(0); + case INT64 -> row -> row.getLong(0); + case FLOAT -> row -> row.getFloat(0); + case DOUBLE -> row -> row.getDouble(0); + case BOOLEAN -> row -> row.getBoolean(0); + case TEXT -> row -> row.getString(0); + default -> row -> 0; + }; + + public static final TypeService ROW_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> (collector, time, value) -> collector.putInt(time, (Integer) value); + case INT64 -> (collector, time, value) -> collector.putLong(time, (Long) value); + case FLOAT -> (collector, time, value) -> collector.putFloat(time, (Float) value); + case DOUBLE -> (collector, time, value) -> collector.putDouble(time, (Double) value); + case BOOLEAN -> (collector, time, value) -> collector.putBoolean(time, (Boolean) value); + default -> (collector, time, value) -> {}; + }; + + public static final TypeService COLUMN_NUMERIC_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> Column::getInt; + case INT64 -> Column::getLong; + case FLOAT -> Column::getFloat; + case DOUBLE -> Column::getDouble; + case BOOLEAN -> (column, index) -> column.getBoolean(index) ? 1.0D : 0.0D; + default -> + (column, index) -> { + throw new IllegalArgumentException( + "Unsupported data type: " + type.getTypeEnum()); + }; + }; + + private static final IndexedNumericRowReader[] INDEXED_NUMERIC_ROW_READERS = + new IndexedNumericRowReader[Type.values().length]; + private static final RowValueReader[] ROW_VALUE_READERS = + new RowValueReader[Type.values().length]; + private static final RowValueWriter[] ROW_VALUE_WRITERS = + new RowValueWriter[Type.values().length]; + + static { + for (Type udfType : Type.values()) { + org.apache.tsfile.read.common.type.Type readType = toReadType(udfType); + INDEXED_NUMERIC_ROW_READERS[udfType.ordinal()] = + INDEXED_NUMERIC_ROW_READER_SERVICE.call(readType); + ROW_VALUE_READERS[udfType.ordinal()] = ROW_VALUE_READER_SERVICE.call(readType); + ROW_VALUE_WRITERS[udfType.ordinal()] = ROW_VALUE_WRITER_SERVICE.call(readType); + } + } + + public static IndexedNumericRowReader indexedNumericRowReader(Type type) { + return INDEXED_NUMERIC_ROW_READERS[type.ordinal()]; + } + + public static RowValueReader rowValueReader(Type type) { + return ROW_VALUE_READERS[type.ordinal()]; + } + + public static RowValueWriter rowValueWriter(Type type) { + return ROW_VALUE_WRITERS[type.ordinal()]; + } + + public static TypeService numericService( + T intService, T longService, T floatService, T doubleService, T unsupportedService) { + return type -> + switch (type.getTypeEnum()) { + case INT32 -> intService; + case INT64 -> longService; + case FLOAT -> floatService; + case DOUBLE -> doubleService; + default -> unsupportedService; + }; + } + + public static TypeService scalarService( + T booleanService, + T intService, + T longService, + T floatService, + T doubleService, + T textService, + T unsupportedService) { + return type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> booleanService; + case INT32 -> intService; + case INT64 -> longService; + case FLOAT -> floatService; + case DOUBLE -> doubleService; + case TEXT, STRING -> textService; + default -> unsupportedService; + }; + } + + public static final TypeService NUMERIC_WINDOW_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (times, values, collector) -> { + for (int i = 0; i < times.length; i++) { + collector.putInt(times[i], (int) Math.round(values[i])); + } + }; + case INT64 -> + (times, values, collector) -> { + for (int i = 0; i < times.length; i++) { + collector.putLong(times[i], Math.round(values[i])); + } + }; + case FLOAT -> + (times, values, collector) -> { + for (int i = 0; i < times.length; i++) { + collector.putFloat(times[i], (float) values[i]); + } + }; + case DOUBLE -> + (times, values, collector) -> { + for (int i = 0; i < times.length; i++) { + collector.putDouble(times[i], values[i]); + } + }; + default -> + (times, values, collector) -> { + throw new NoNumberException(); + }; + }; + + public static final TypeService NUMERIC_VALUE_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> (time, value, collector) -> collector.putInt(time, (int) value); + case INT64 -> (time, value, collector) -> collector.putLong(time, (long) value); + case FLOAT -> (time, value, collector) -> collector.putFloat(time, (float) value); + case DOUBLE -> (time, value, collector) -> collector.putDouble(time, value); + default -> + (time, value, collector) -> { + throw new NoNumberException(); + }; + }; + + public static final TypeService NUMERIC_CAST_WINDOW_WRITER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + (times, values, collector) -> { + for (int i = 0; i < times.length; i++) { + collector.putInt(times[i], (int) values[i]); + } + }; + case INT64 -> + (times, values, collector) -> { + for (int i = 0; i < times.length; i++) { + collector.putLong(times[i], (long) values[i]); + } + }; + case FLOAT -> + (times, values, collector) -> { + for (int i = 0; i < times.length; i++) { + collector.putFloat(times[i], (float) values[i]); + } + }; + case DOUBLE -> + (times, values, collector) -> { + for (int i = 0; i < times.length; i++) { + collector.putDouble(times[i], values[i]); + } + }; + default -> + (times, values, collector) -> { + throw new NoNumberException(); + }; + }; + + public static org.apache.tsfile.read.common.type.Type toReadType(Type type) { + return org.apache.tsfile.read.common.type.Type.fromTsDataType( + TSDataType.getTsDataType(type.getType())); + } + + private TypeServices() {} + + @FunctionalInterface + public interface NumericRowReader { + double read(Row row) throws IOException, NoNumberException; + } + + @FunctionalInterface + public interface IndexedNumericRowReader { + double read(Row row, int index) throws IOException, NoNumberException; + } + + @FunctionalInterface + public interface NumericRowWriter { + void write(Row row, PointCollector collector) throws IOException, NoNumberException; + } + + @FunctionalInterface + public interface RowValueReader { + Object read(Row row) throws IOException; + } + + @FunctionalInterface + public interface RowValueWriter { + void write(PointCollector collector, long time, Object value) throws IOException; + } + + @FunctionalInterface + public interface ColumnNumericReader { + double read(Column column, int index); + } + + @FunctionalInterface + public interface NumericWindowWriter { + void write(long[] times, double[] values, PointCollector collector) + throws IOException, NoNumberException; + } + + @FunctionalInterface + public interface NumericValueWriter { + void write(long time, double value, PointCollector collector) + throws IOException, NoNumberException; + } +} diff --git a/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java b/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java index f947b1651adf..76b0831060f4 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java @@ -50,34 +50,11 @@ private Util() { * @throws NoNumberException when getting a no number datatype */ public static double getValueAsDouble(Row row, int index) throws IOException, NoNumberException { - double ans = 0; try { - switch (row.getDataType(index)) { - case INT32: - ans = row.getInt(index); - break; - case INT64: - ans = row.getLong(index); - break; - case FLOAT: - ans = row.getFloat(index); - break; - case DOUBLE: - ans = row.getDouble(index); - break; - case DATE: - case BLOB: - case BOOLEAN: - case STRING: - case TEXT: - case TIMESTAMP: - default: - throw new NoNumberException(); - } + return TypeServices.indexedNumericRowReader(row.getDataType(index)).read(row, index); } catch (IOException e) { throw new IOException(LibraryUdfMessages.FAIL_TO_GET_DATA_TYPE_IN_ROW + row.getTime(), e); } - return ans; } /** @@ -98,34 +75,7 @@ public static double getValueAsDouble(Row row) throws IOException, NoNumberExcep * @return value from 0th column from Row */ public static Object getValueAsObject(Row row) throws IOException { - Object ans = 0; - switch (row.getDataType(0)) { - case INT32: - ans = row.getInt(0); - break; - case INT64: - ans = row.getLong(0); - break; - case FLOAT: - ans = row.getFloat(0); - break; - case DOUBLE: - ans = row.getDouble(0); - break; - case BOOLEAN: - ans = row.getBoolean(0); - break; - case TEXT: - ans = row.getString(0); - break; - case BLOB: - case STRING: - case DATE: - case TIMESTAMP: - default: - break; - } - return ans; + return TypeServices.rowValueReader(row.getDataType(0)).read(row); } /** @@ -137,30 +87,7 @@ public static Object getValueAsObject(Row row) throws IOException { * @param o value in Object type */ public static void putValue(PointCollector pc, Type type, long t, Object o) throws IOException { - switch (type) { - case INT32: - pc.putInt(t, (Integer) o); - break; - case INT64: - pc.putLong(t, (Long) o); - break; - case FLOAT: - pc.putFloat(t, (Float) o); - break; - case DOUBLE: - pc.putDouble(t, (Double) o); - break; - case BOOLEAN: - pc.putBoolean(t, (Boolean) o); - break; - case DATE: - case TIMESTAMP: - case TEXT: - case STRING: - case BLOB: - default: - break; - } + TypeServices.rowValueWriter(type).write(pc, t, o); } /** diff --git a/pom.xml b/pom.xml index 7d4b3d74a426..0c0dfdb4cc62 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ example library-udf external-service-impl + architecture-test 1.7