Skip to content

Migrate to Jackson 3.x APIs #5703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
reta wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Migrate to Jackson 3.x APIs #5703

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions core/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,10 @@ dependencies {
api group: 'org.apache.commons', name: 'commons-text', version: "${commons_text_version}"
api group: 'com.facebook.presto', name: 'presto-matching', version: '0.240'
api group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
api "tools.jackson.core:jackson-core:${versions.jackson3}"
api "tools.jackson.core:jackson-databind:${versions.jackson3_databind}"
api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}"
api "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}"
api "tools.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson3}"
api group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
api group: 'com.tdunning', name: 't-digest', version: '3.3'
api "net.minidev:json-smart:${versions.json_smart}"
Expand Down
6 changes: 3 additions & 3 deletions .../src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@
import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE;
import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
Expand All @@ -25,6 +23,8 @@
import org.apache.calcite.sql.type.SqlReturnTypeInference;
import org.opensearch.sql.expression.function.ImplementorUDF;
import org.opensearch.sql.expression.function.UDFOperandMetadata;
import tools.jackson.core.JacksonException;
import tools.jackson.databind.JsonNode;

public class JsonAppendFunctionImpl extends ImplementorUDF {
public JsonAppendFunctionImpl() {
Expand Down Expand Up @@ -53,7 +53,7 @@ public Expression implement(
}
}

public static Object eval(Object... args) throws JsonProcessingException {
public static Object eval(Object... args) throws JacksonException {
String jsonStr = (String) args[0];
List<Object> keys = Arrays.asList(args).subList(1, args.length);
if (keys.size() % 2 != 0) {
Expand Down
4 changes: 2 additions & 2 deletions .../src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE;
import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*;

import com.fasterxml.jackson.core.JsonProcessingException;
import java.util.Arrays;
import java.util.List;
import org.apache.calcite.adapter.enumerable.NotNullImplementor;
Expand All @@ -23,6 +22,7 @@
import org.apache.calcite.sql.type.SqlReturnTypeInference;
import org.opensearch.sql.expression.function.ImplementorUDF;
import org.opensearch.sql.expression.function.UDFOperandMetadata;
import tools.jackson.core.JacksonException;

public class JsonDeleteFunctionImpl extends ImplementorUDF {
public JsonDeleteFunctionImpl() {
Expand Down Expand Up @@ -51,7 +51,7 @@ public Expression implement(
}
}

public static Object eval(Object... args) throws JsonProcessingException {
public static Object eval(Object... args) throws JacksonException {
List<Object> jsonPaths = Arrays.asList(args).subList(1, args.length);
String[] pathSpecs =
jsonPaths.stream()
Expand Down
6 changes: 3 additions & 3 deletions .../src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@
import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE;
import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
Expand All @@ -25,6 +23,8 @@
import org.apache.calcite.sql.type.SqlReturnTypeInference;
import org.opensearch.sql.expression.function.ImplementorUDF;
import org.opensearch.sql.expression.function.UDFOperandMetadata;
import tools.jackson.core.JacksonException;
import tools.jackson.databind.JsonNode;

public class JsonExtendFunctionImpl extends ImplementorUDF {
public JsonExtendFunctionImpl() {
Expand Down Expand Up @@ -53,7 +53,7 @@ public Expression implement(
}
}

public static Object eval(Object... args) throws JsonProcessingException {
public static Object eval(Object... args) throws JacksonException {
String jsonStr = (String) args[0];
List<Object> keys = Arrays.asList(args).subList(1, args.length);
if (keys.size() % 2 != 0) {
Expand Down
19 changes: 10 additions & 9 deletions .../main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,6 @@
import static java.util.stream.Collectors.toMap;
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
Expand All @@ -33,6 +29,11 @@
import org.apache.calcite.sql.type.SqlTypeName;
import org.opensearch.sql.expression.function.ImplementorUDF;
import org.opensearch.sql.expression.function.UDFOperandMetadata;
import tools.jackson.core.JacksonException;
import tools.jackson.core.JsonParser;
import tools.jackson.core.JsonToken;
import tools.jackson.core.ObjectReadContext;
import tools.jackson.core.json.JsonFactory;

/**
* UDF which extract all the fields from JSON to a MAP. Items are collected from input JSON and
Expand Down Expand Up @@ -106,7 +107,7 @@ private static Map<String, Object> parseJson(String jsonStr) {
Map<String, Object> resultMap = new HashMap<>();
Stack<String> pathStack = new Stack<>();

try (JsonParser parser = JSON_FACTORY.createParser(jsonStr)) {
try (JsonParser parser = JSON_FACTORY.createParser(ObjectReadContext.empty(), jsonStr)) {
JsonToken token;

while ((token = parser.nextToken()) != null) {
Expand All @@ -131,7 +132,7 @@ private static Map<String, Object> parseJson(String jsonStr) {
}
break;

case FIELD_NAME:
case PROPERTY_NAME:
String fieldName = parser.currentName();
pathStack.push(fieldName);
break;
Expand All @@ -158,7 +159,7 @@ private static Map<String, Object> parseJson(String jsonStr) {
break;
}
}
} catch (IOException e) {
} catch (JacksonException e) {
// ignore exception, and current result will be returned
}
return resultMap;
Expand Down Expand Up @@ -188,7 +189,7 @@ private static boolean isInArray(List<String> path) {
return path.size() >= 1 && path.getLast().equals(ARRAY_SUFFIX);
}

private static Object extractValue(JsonParser parser, JsonToken token) throws IOException {
private static Object extractValue(JsonParser parser, JsonToken token) throws JacksonException {
switch (token) {
case VALUE_STRING:
return parser.getValueAsString();
Expand All @@ -207,7 +208,7 @@ private static Object extractValue(JsonParser parser, JsonToken token) throws IO
}
}

private static Object getIntValue(JsonParser parser) throws IOException {
private static Object getIntValue(JsonParser parser) throws JacksonException {
if (parser.getNumberType() == JsonParser.NumberType.INT) {
return parser.getIntValue();
} else if (parser.getNumberType() == JsonParser.NumberType.LONG) {
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE;
import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*;

import com.fasterxml.jackson.databind.JsonNode;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
Expand All @@ -24,6 +23,7 @@
import org.apache.calcite.sql.type.SqlReturnTypeInference;
import org.opensearch.sql.expression.function.ImplementorUDF;
import org.opensearch.sql.expression.function.UDFOperandMetadata;
import tools.jackson.databind.JsonNode;

public class JsonSetFunctionImpl extends ImplementorUDF {
public JsonSetFunctionImpl() {
Expand Down
4 changes: 2 additions & 2 deletions core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@

package org.opensearch.sql.expression.function.jsonUDF;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import java.util.ArrayList;
import java.util.List;
import tools.jackson.databind.JsonNode;
import tools.jackson.databind.ObjectMapper;

public class JsonUtils {
static ObjectMapper objectMapper = new ObjectMapper();
Expand Down
14 changes: 7 additions & 7 deletions core/src/main/java/org/opensearch/sql/utils/JsonUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@
import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL;
import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
Expand All @@ -27,6 +24,9 @@
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.data.model.ExprValueUtils;
import org.opensearch.sql.exception.SemanticCheckException;
import tools.jackson.core.JacksonException;
import tools.jackson.databind.JsonNode;
import tools.jackson.databind.ObjectMapper;

@UtilityClass
public class JsonUtils {
Expand All @@ -46,7 +46,7 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) {
try {
objectMapper.readTree(jsonExprValue.stringValue());
return LITERAL_TRUE;
} catch (JsonProcessingException e) {
} catch (JacksonException e) {
return LITERAL_FALSE;
}
}
Expand All @@ -71,7 +71,7 @@ public static ExprValue castJson(ExprValue json) {
JsonNode jsonNode;
try {
jsonNode = objectMapper.readTree(json.stringValue());
} catch (JsonProcessingException e) {
} catch (JacksonException e) {
final String errorFormat = "JSON string '%s' is not valid. Error details: %s";
throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e);
}
Expand All @@ -90,13 +90,13 @@ private static ExprValue processJsonNode(JsonNode jsonNode) {
return new ExprCollectionValue(elements);
case OBJECT:
Map<String, ExprValue> values = new LinkedHashMap<>();
for (var iter = jsonNode.fields(); iter.hasNext(); ) {
for (var iter = jsonNode.properties().iterator(); iter.hasNext(); ) {
Map.Entry<String, JsonNode> entry = iter.next();
values.put(entry.getKey(), processJsonNode(entry.getValue()));
}
return ExprTupleValue.fromExprValueMap(values);
case STRING:
return new ExprStringValue(jsonNode.asText());
return new ExprStringValue(jsonNode.asString());
case NUMBER:
if (jsonNode.isFloatingPointNumber()) {
return new ExprDoubleValue(jsonNode.asDouble());
Expand Down
37 changes: 20 additions & 17 deletions core/src/main/java/org/opensearch/sql/utils/YamlFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@
package org.opensearch.sql.utils;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
import tools.jackson.core.JacksonException;
import tools.jackson.databind.ObjectMapper;
import tools.jackson.dataformat.yaml.YAMLFactory;
import tools.jackson.dataformat.yaml.YAMLFactoryBuilder;
import tools.jackson.dataformat.yaml.YAMLMapper;
import tools.jackson.dataformat.yaml.YAMLWriteFeature;

/**
* YAML formatter utility class. Attributes are sorted alphabetically for consistent output. Check
Expand All @@ -21,18 +22,20 @@ public class YamlFormatter {
private static final ObjectMapper YAML_MAPPER;

static {
YAMLFactory yamlFactory = new YAMLFactory();
yamlFactory.disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER);
yamlFactory.enable(YAMLGenerator.Feature.USE_PLATFORM_LINE_BREAKS);
yamlFactory.enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE);
yamlFactory.enable(YAMLGenerator.Feature.MINIMIZE_QUOTES); // Enable smart quoting
yamlFactory.enable(
YAMLGenerator.Feature.ALWAYS_QUOTE_NUMBERS_AS_STRINGS); // Quote numeric strings
yamlFactory.enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR);
YAML_MAPPER = new ObjectMapper(yamlFactory);
final YAMLFactoryBuilder builder = new YAMLFactoryBuilder(new YAMLFactory());
builder.disable(YAMLWriteFeature.WRITE_DOC_START_MARKER);
builder.enable(YAMLWriteFeature.LITERAL_BLOCK_STYLE);
builder.enable(YAMLWriteFeature.MINIMIZE_QUOTES); // Enable smart quoting
builder.enable(YAMLWriteFeature.ALWAYS_QUOTE_NUMBERS_AS_STRINGS); // Quote numeric strings
builder.enable(YAMLWriteFeature.INDENT_ARRAYS_WITH_INDICATOR);

YAML_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL);
YAML_MAPPER.configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false);
YAML_MAPPER =
YAMLMapper.builder(builder.build())
.changeDefaultPropertyInclusion(
incl -> incl.withValueInclusion(JsonInclude.Include.NON_NULL))
.changeDefaultPropertyInclusion(
incl -> incl.withContentInclusion(JsonInclude.Include.NON_NULL))
.build();
}

/**
Expand All @@ -44,7 +47,7 @@ public class YamlFormatter {
public static String formatToYaml(Object object) {
try {
return YAML_MAPPER.writeValueAsString(object);
} catch (JsonProcessingException e) {
} catch (JacksonException e) {
throw new RuntimeException("Failed to format object to YAML", e);
}
}
Expand Down
2 changes: 1 addition & 1 deletion core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*;

import com.fasterxml.jackson.databind.JsonNode;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
Expand All @@ -22,6 +21,7 @@
import org.opensearch.sql.expression.function.jsonUDF.JsonDeleteFunctionImpl;
import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl;
import org.opensearch.sql.expression.function.jsonUDF.JsonUtils;
import tools.jackson.databind.JsonNode;

@ExtendWith(MockitoExtension.class)
public class JsonFunctionsTest {
Expand Down
31 changes: 17 additions & 14 deletions core/src/test/java/org/opensearch/sql/utils/YamlFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@

package org.opensearch.sql.utils;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
import tools.jackson.core.JacksonException;
import tools.jackson.databind.ObjectMapper;
import tools.jackson.databind.SerializationFeature;
import tools.jackson.dataformat.yaml.YAMLFactory;
import tools.jackson.dataformat.yaml.YAMLFactoryBuilder;
import tools.jackson.dataformat.yaml.YAMLMapper;
import tools.jackson.dataformat.yaml.YAMLWriteFeature;

/**
* YAML formatter utility class. Attributes are sorted alphabetically for consistent output. Check
Expand All @@ -20,23 +22,24 @@ public class YamlFormatter {
private static final ObjectMapper YAML_MAPPER = initObjectMapper();

private static ObjectMapper initObjectMapper() {
YAMLFactory yamlFactory = new YAMLFactory();
yamlFactory.disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER);
yamlFactory.enable(YAMLGenerator.Feature.MINIMIZE_QUOTES); // Enable smart quoting
yamlFactory.enable(
YAMLGenerator.Feature.ALWAYS_QUOTE_NUMBERS_AS_STRINGS); // Quote numeric strings
yamlFactory.enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR);
final YAMLFactoryBuilder builder = new YAMLFactoryBuilder(new YAMLFactory());
builder.disable(YAMLWriteFeature.WRITE_DOC_START_MARKER);
builder.enable(YAMLWriteFeature.MINIMIZE_QUOTES); // Enable smart quoting
builder.enable(YAMLWriteFeature.ALWAYS_QUOTE_NUMBERS_AS_STRINGS); // Quote numeric strings
builder.enable(YAMLWriteFeature.INDENT_ARRAYS_WITH_INDICATOR);

ObjectMapper mapper = new ObjectMapper(yamlFactory);
mapper.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
ObjectMapper mapper =
YAMLMapper.builder(builder.build())
.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
.build();
return mapper;
}

/** Formats any object into YAML. It will always use LF as line break regardless of OS. */
public static String formatToYaml(Object object) {
try {
return YAML_MAPPER.writer().withDefaultPrettyPrinter().writeValueAsString(object);
} catch (JsonProcessingException e) {
} catch (JacksonException e) {
throw new RuntimeException("Failed to format object to YAML", e);
}
}
Expand Down
Loading
Loading