From e8283558569f968c09e4c79e3f2e01ae7a7514e1 Mon Sep 17 00:00:00 2001 From: Sebastian Gruza Date: Sun, 20 Sep 2026 16:48:44 +0000 Subject: [PATCH 1/8] feat(client): add the DECIMAL (BigDecimal) property data type Companion of apache/hugegraph#3209 (issue apache/hugegraph#3206): the server gains `DataType.DECIMAL`, an exact arbitrary-precision property type meant for amounts that go through batch `update_strategies: SUM`. This makes the toolchain able to declare and carry such values. hugegraph-client - `DataType.DECIMAL(12, "decimal", BigDecimal.class)` with `isDecimal()` and `valueToDecimal()` (same conversion rules as the server) in the public enum and in the direct-serializer copy; `PropertyKey.Builder .asDecimal()`. - A `BigDecimal` is serialized as a plain string ("1.10", never "1.1E+2") in request bodies and query parameters: a JSON number is parsed as a double on the server side and would lose precision and trailing zeros. Values read back are the plain string the server returns; `new BigDecimal(String)` restores them exactly. - `BytesBuffer.writeProperty` writes the server's DECIMAL layout (unscaled two's-complement bytes + scale) for the direct loaders. - `serializer/direct/struct/DataType` could never be initialised (its code table was filled before it was created); ordered the statics. - Tests: `DecimalDataTypeTest` (unit) and `DecimalPropertyApiTest` (API, skipped with `Assume` while the CI server has no DECIMAL yet). Two `BatchUpdateElementApiTest` assertions now accept the message the server produces once it normalises batch values to the property type. hugegraph-loader / hugegraph-spark-connector - `decimal` columns convert to `BigDecimal` (`DataTypeUtilTest`). hugegraph-hubble - Groovy schema export emits `.asDecimal()`. --- .../apache/hugegraph/client/RestClient.java | 8 + .../serializer/BigDecimalSerializer.java | 46 ++++ .../serializer/direct/struct/DataType.java | 43 +++- .../serializer/direct/util/BytesBuffer.java | 8 + .../structure/constant/DataType.java | 43 +++- .../structure/schema/PropertyKey.java | 8 + .../org/apache/hugegraph/util/JsonUtil.java | 14 ++ .../apache/hugegraph/api/ApiTestSuite.java | 1 + .../api/BatchUpdateElementApiTest.java | 8 +- .../hugegraph/api/DecimalPropertyApiTest.java | 225 ++++++++++++++++++ .../hugegraph/unit/DecimalDataTypeTest.java | 135 +++++++++++ .../apache/hugegraph/unit/UnitTestSuite.java | 1 + .../schema/GroovySchemaCompatibility.java | 1 + .../hugegraph/loader/util/DataTypeUtil.java | 21 ++ .../loader/test/unit/DataTypeUtilTest.java | 92 +++++++ .../loader/test/unit/UnitTestSuite.java | 1 + .../spark/connector/utils/DataTypeUtils.java | 18 ++ 17 files changed, 667 insertions(+), 6 deletions(-) create mode 100644 hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java create mode 100644 hugegraph-client/src/test/java/org/apache/hugegraph/api/DecimalPropertyApiTest.java create mode 100644 hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java create mode 100644 hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/client/RestClient.java b/hugegraph-client/src/main/java/org/apache/hugegraph/client/RestClient.java index ca1dcfad3..7c0f7bbf0 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/client/RestClient.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/client/RestClient.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.client; +import java.math.BigDecimal; import java.util.Map; import org.apache.hugegraph.exception.ServerException; @@ -25,9 +26,11 @@ import org.apache.hugegraph.rest.RestClientConfig; import org.apache.hugegraph.rest.RestHeaders; import org.apache.hugegraph.rest.RestResult; +import org.apache.hugegraph.serializer.BigDecimalSerializer; import org.apache.hugegraph.serializer.PathDeserializer; import org.apache.hugegraph.structure.graph.Path; import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.JsonUtilCommon; import org.apache.hugegraph.util.VersionUtil; import org.apache.hugegraph.util.VersionUtil.Version; @@ -49,6 +52,11 @@ public class RestClient extends AbstractRestClient { SimpleModule module = new SimpleModule(); module.addDeserializer(Path.class, new PathDeserializer()); RestResult.registerModule(module); + + // Request bodies: a decimal goes as a plain string, see JsonUtil + SimpleModule decimals = new SimpleModule(); + decimals.addSerializer(BigDecimal.class, new BigDecimalSerializer()); + JsonUtilCommon.registerModule(decimals); } public RestClient(String url, String username, String password, int timeout) { diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java new file mode 100644 index 000000000..cd30e933f --- /dev/null +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.serializer; + +import java.io.IOException; +import java.math.BigDecimal; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; + +/** + * Write a BigDecimal as a plain decimal string ("1.10", never "1.1E+2"). + * A JSON number would be parsed as a double on the server side and lose + * both precision and trailing zeros; a string reaches a DECIMAL property + * key exactly, which is the point of that data type. + */ +public class BigDecimalSerializer extends StdSerializer { + + private static final long serialVersionUID = 1L; + + public BigDecimalSerializer() { + super(BigDecimal.class); + } + + @Override + public void serialize(BigDecimal value, JsonGenerator generator, + SerializerProvider provider) throws IOException { + generator.writeString(value.toPlainString()); + } +} diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/struct/DataType.java b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/struct/DataType.java index 00291b728..216a5567a 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/struct/DataType.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/struct/DataType.java @@ -17,6 +17,8 @@ package org.apache.hugegraph.serializer.direct.struct; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.Date; import java.util.UUID; @@ -42,18 +44,20 @@ public enum DataType { TEXT(8, "text", String.class), //BLOB(9, "blob", Blob.class), DATE(10, "date", Date.class), - UUID(11, "uuid", UUID.class); + UUID(11, "uuid", UUID.class), + DECIMAL(12, "decimal", BigDecimal.class); private final byte code; private final String name; private final Class clazz; + // Must be initialized before the static block that fills it + static Table, Byte, DataType> TABLE = HashBasedTable.create(); + static { register(DataType.class); } - static Table, Byte, DataType> TABLE = HashBasedTable.create(); - static void register(Class clazz) { Object enums; try { @@ -125,6 +129,10 @@ public boolean isUUID() { return this == DataType.UUID; } + public boolean isDecimal() { + return this == DataType.DECIMAL; + } + public Number valueToNumber(V value) { if (!(this.isNumber() && value instanceof Number)) { return null; @@ -192,6 +200,35 @@ public UUID valueToUUID(V value) { return null; } + /** + * Convert a value to BigDecimal the same way the server does: BigDecimal + * as is, integral numbers exactly, any other Number and a decimal string + * through their decimal representation. + * + * @return the BigDecimal, or null if the value can't be a decimal + */ + public BigDecimal valueToDecimal(V value) { + if (!this.isDecimal()) { + return null; + } + if (value instanceof BigDecimal) { + return (BigDecimal) value; + } else if (value instanceof BigInteger) { + return new BigDecimal((BigInteger) value); + } else if (value instanceof Byte || value instanceof Short || + value instanceof Integer || value instanceof Long) { + return BigDecimal.valueOf(((Number) value).longValue()); + } else if (!(value instanceof Number) && !(value instanceof String)) { + return null; + } + try { + return new BigDecimal(value.toString().trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "Can't read '%s' as decimal", value)); + } + } + public static DataType fromClass(Class clazz) { for (DataType type : DataType.values()) { if (type.clazz() == clazz) { diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/util/BytesBuffer.java b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/util/BytesBuffer.java index 054f24bbe..f4652194a 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/util/BytesBuffer.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/util/BytesBuffer.java @@ -20,6 +20,7 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Arrays; +import java.math.BigDecimal; import java.util.Date; import java.util.UUID; @@ -750,6 +751,13 @@ public void writeProperty(DataType dataType, Object value) { this.writeLong(uuid.getMostSignificantBits()); this.writeLong(uuid.getLeastSignificantBits()); break; + case DECIMAL: + // Same layout as the server: unscaled two's-complement + // bytes followed by the scale, exact for any precision + BigDecimal decimal = dataType.valueToDecimal(value); + this.writeBytes(decimal.unscaledValue().toByteArray()); + this.writeVInt(decimal.scale()); + break; default: //this.writeBytes(KryoUtil.toKryoWithType(value)); break; diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java b/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java index fe68c77a2..57b89b994 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java @@ -18,6 +18,8 @@ package org.apache.hugegraph.structure.constant; import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.Date; import java.util.UUID; @@ -33,7 +35,12 @@ public enum DataType { TEXT(8, "text", String.class), BLOB(9, "blob", byte[].class), DATE(10, "date", Date.class), - UUID(11, "uuid", UUID.class); + UUID(11, "uuid", UUID.class), + /* + * Arbitrary-precision decimal (java.math.BigDecimal), stored exactly by + * the server; sent and received as a plain decimal string in JSON + */ + DECIMAL(12, "decimal", BigDecimal.class); private final byte code; private final String name; @@ -71,6 +78,40 @@ public boolean isUUID() { return this == DataType.UUID; } + public boolean isDecimal() { + return this == DataType.DECIMAL; + } + + /** + * Convert a value to BigDecimal the same way the server does: BigDecimal + * as is, integral numbers exactly, any other Number and a decimal string + * through their decimal representation. + * + * @return the BigDecimal, or null if the value can't be a decimal + * @throws IllegalArgumentException if the string is not a decimal number + */ + public BigDecimal valueToDecimal(V value) { + if (!this.isDecimal()) { + return null; + } + if (value instanceof BigDecimal) { + return (BigDecimal) value; + } else if (value instanceof BigInteger) { + return new BigDecimal((BigInteger) value); + } else if (value instanceof Byte || value instanceof Short || + value instanceof Integer || value instanceof Long) { + return BigDecimal.valueOf(((Number) value).longValue()); + } else if (!(value instanceof Number) && !(value instanceof String)) { + return null; + } + try { + return new BigDecimal(value.toString().trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "Can't read '%s' as decimal", value)); + } + } + public boolean isBoolean() { return this == BOOLEAN; } diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/structure/schema/PropertyKey.java b/hugegraph-client/src/main/java/org/apache/hugegraph/structure/schema/PropertyKey.java index b690c1e80..bc3d574da 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/structure/schema/PropertyKey.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/structure/schema/PropertyKey.java @@ -112,6 +112,8 @@ public interface Builder extends SchemaBuilder { Builder asLong(); + Builder asDecimal(); + Builder cardinality(Cardinality cardinality); Builder valueSingle(); @@ -266,6 +268,12 @@ public Builder asLong() { return this; } + @Override + public Builder asDecimal() { + this.propertyKey.dataType = DataType.DECIMAL; + return this; + } + @Override public Builder cardinality(Cardinality cardinality) { this.propertyKey.cardinality = cardinality; diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/util/JsonUtil.java b/hugegraph-client/src/main/java/org/apache/hugegraph/util/JsonUtil.java index 8222fdea8..c34acbcc2 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/util/JsonUtil.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/util/JsonUtil.java @@ -18,18 +18,32 @@ package org.apache.hugegraph.util; import java.io.IOException; +import java.math.BigDecimal; import org.apache.hugegraph.rest.SerializeException; +import org.apache.hugegraph.serializer.BigDecimalSerializer; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; public final class JsonUtil { private static final ObjectMapper MAPPER = new ObjectMapper(); + static { + /* + * A DECIMAL property value travels as a plain string ("1.10"): a + * JSON number is read as a double on the other side. The same + * serializer is registered for request bodies in RestClient. + */ + SimpleModule module = new SimpleModule(); + module.addSerializer(BigDecimal.class, new BigDecimalSerializer()); + MAPPER.registerModule(module); + } + public static void registerModule(Module module) { MAPPER.registerModule(module); } diff --git a/hugegraph-client/src/test/java/org/apache/hugegraph/api/ApiTestSuite.java b/hugegraph-client/src/test/java/org/apache/hugegraph/api/ApiTestSuite.java index c9edba73f..57c0e74e9 100644 --- a/hugegraph-client/src/test/java/org/apache/hugegraph/api/ApiTestSuite.java +++ b/hugegraph-client/src/test/java/org/apache/hugegraph/api/ApiTestSuite.java @@ -59,6 +59,7 @@ VertexApiTest.class, EdgeApiTest.class, BatchUpdateElementApiTest.class, + DecimalPropertyApiTest.class, GremlinApiTest.class, VariablesApiTest.class, diff --git a/hugegraph-client/src/test/java/org/apache/hugegraph/api/BatchUpdateElementApiTest.java b/hugegraph-client/src/test/java/org/apache/hugegraph/api/BatchUpdateElementApiTest.java index 4604f6860..fc0678a19 100644 --- a/hugegraph-client/src/test/java/org/apache/hugegraph/api/BatchUpdateElementApiTest.java +++ b/hugegraph-client/src/test/java/org/apache/hugegraph/api/BatchUpdateElementApiTest.java @@ -329,8 +329,10 @@ public void testVertexInvalidUpdateStrategy() { Assert.assertThrows(ServerException.class, () -> { vertexAPI.update(req5); }, e -> { + // The new value is a String before, a Date after, the server + // started normalising batch values to the property's data type String expect = "Property type must be Set or List for " + - "strategy INTERSECTION, but got type Date, String"; + "strategy INTERSECTION, but got type Date, "; Assert.assertContains(expect, e.getMessage()); }); @@ -621,8 +623,10 @@ public void testEdgeInvalidUpdateStrategy() { Assert.assertThrows(ServerException.class, () -> { edgeAPI.update(req5); }, e -> { + // The new value is a String before, a Date after, the server + // started normalising batch values to the property's data type String expect = "Property type must be Set or List for " + - "strategy INTERSECTION, but got type Date, String"; + "strategy INTERSECTION, but got type Date, "; Assert.assertContains(expect, e.getMessage()); }); diff --git a/hugegraph-client/src/test/java/org/apache/hugegraph/api/DecimalPropertyApiTest.java b/hugegraph-client/src/test/java/org/apache/hugegraph/api/DecimalPropertyApiTest.java new file mode 100644 index 000000000..cfcd30b27 --- /dev/null +++ b/hugegraph-client/src/test/java/org/apache/hugegraph/api/DecimalPropertyApiTest.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.api; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.hugegraph.exception.ServerException; +import org.apache.hugegraph.structure.constant.DataType; +import org.apache.hugegraph.structure.constant.T; +import org.apache.hugegraph.structure.graph.BatchVertexRequest; +import org.apache.hugegraph.structure.graph.UpdateStrategy; +import org.apache.hugegraph.structure.graph.Vertex; +import org.apache.hugegraph.structure.gremlin.ResultSet; +import org.apache.hugegraph.structure.schema.PropertyKey; +import org.apache.hugegraph.structure.schema.VertexLabel; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.testutil.Utils; +import org.junit.After; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +/** + * DECIMAL property keys: exact values across create/read/batch-SUM. The + * whole class is skipped against a server that has no DECIMAL data type yet. + */ +public class DecimalPropertyApiTest extends BaseApiTest { + + private static final String UINT256_MAX = + "115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"; + + @BeforeClass + public static void prepareSchema() { + PropertyKey amount = new PropertyKey.BuilderImpl("amount", null) + .asDecimal().build(); + try { + propertyKeyAPI.create(amount); + } catch (ServerException e) { + Assume.assumeTrue("The server has no DECIMAL data type: " + + e.getMessage(), false); + } + propertyKeyAPI.create(new PropertyKey.BuilderImpl("name", null) + .asText().build()); + vertexLabelAPI.create(new VertexLabel.BuilderImpl("account", null) + .properties("name", "amount") + .primaryKeys("name") + .nullableKeys("amount") + .build()); + } + + @Override + @After + public void teardown() { + vertexAPI.list(-1).results().forEach(v -> vertexAPI.delete(v.id())); + } + + @Test + public void testPropertyKeyAsDecimal() { + PropertyKey amount = propertyKeyAPI.get("amount"); + Assert.assertEquals(DataType.DECIMAL, amount.dataType()); + Assert.assertEquals(DataType.DECIMAL, + schema().getPropertyKey("amount").dataType()); + } + + @Test + public void testDecimalVertexRoundTrip() { + Vertex account = new Vertex("account"); + account.property("name", "alice"); + account.property("amount", new BigDecimal("12345678901234567890.10")); + Object id = vertexAPI.create(account).id(); + + // The value comes back as the plain decimal string, exactly + Vertex vertex = vertexAPI.get(id); + Assert.assertEquals("12345678901234567890.10", + vertex.property("amount")); + Assert.assertEquals(new BigDecimal("12345678901234567890.10"), + new BigDecimal((String) vertex.property("amount"))); + + // Also through the driver and Gremlin + Vertex byDriver = graph().getVertex(id); + Assert.assertEquals("12345678901234567890.10", + byDriver.property("amount")); + ResultSet results = gremlin().gremlin( + "g.V(vid).values('amount')").binding("vid", id).execute(); + Assert.assertEquals(ImmutableList.of("12345678901234567890.10"), + results.data()); + } + + @Test + public void testSchemaManagerAndDriver() { + // The same round trip the way an application uses the client + schema().propertyKey("fee").asDecimal().ifNotExist().create(); + Assert.assertEquals(DataType.DECIMAL, + schema().getPropertyKey("fee").dataType()); + schema().vertexLabel("account").properties("fee") + .nullableKeys("fee").append(); + + Vertex account = graph().addVertex(T.LABEL, "account", + "name", "grace", + "amount", new BigDecimal("1.10"), + "fee", new BigDecimal("0.001")); + Assert.assertEquals("1.10", account.property("amount")); + Assert.assertEquals("0.001", account.property("fee")); + + Vertex delta = new Vertex("account"); + delta.property("name", "grace"); + delta.property("fee", new BigDecimal("0.009")); + Map strategies = ImmutableMap.of( + "fee", UpdateStrategy.SUM); + List updated = graph().updateVertices( + new BatchVertexRequest.Builder() + .vertices(ImmutableList.of(delta)) + .updatingStrategies(strategies) + .createIfNotExist(true).build()); + Assert.assertEquals(1, updated.size()); + Assert.assertEquals("0.010", updated.get(0).property("fee")); + Assert.assertEquals("0.010", + graph().getVertex(account.id()).property("fee")); + } + + @Test + public void testDecimalValueForms() { + // A decimal string, an integral number and uint256 max all stay exact + Map cases = ImmutableMap.of("bob", "0.000000000000000001", + "carol", "42", + "dave", UINT256_MAX); + for (Map.Entry entry : cases.entrySet()) { + Vertex account = new Vertex("account"); + account.property("name", entry.getKey()); + account.property("amount", entry.getValue()); + Object id = vertexAPI.create(account).id(); + Assert.assertEquals(entry.getValue(), + vertexAPI.get(id).property("amount")); + } + + Vertex account = new Vertex("account"); + account.property("name", "erin"); + account.property("amount", 7); + Object id = vertexAPI.create(account).id(); + Assert.assertEquals("7", vertexAPI.get(id).property("amount")); + + Vertex bad = new Vertex("account"); + bad.property("name", "frank"); + bad.property("amount", "1,10"); + Utils.assertResponseError(400, () -> vertexAPI.create(bad)); + } + + @Test + public void testBatchUpdateSumIsExact() { + // Doubles would give 0.30000000000000004 and lose the 18th digit + this.createAccounts(new BigDecimal("0.1")); + List updated = this.sum(new BigDecimal("0.2")); + for (Vertex vertex : updated) { + Assert.assertEquals("0.3", vertex.property("amount")); + } + + this.sum(new BigDecimal("100000000000000000000.000000000000000001")); + this.sum(new BigDecimal("-0.3")); + for (Vertex vertex : vertexAPI.list(-1).results()) { + Assert.assertEquals("100000000000000000000.000000000000000001", + vertex.property("amount")); + } + } + + @Test + public void testDecimalCanNotBeSortKeyOrRangeIndex() { + Utils.assertResponseError(400, () -> { + indexLabelAPI.create( + new org.apache.hugegraph.structure.schema.IndexLabel + .BuilderImpl("accountByAmount", null) + .onV("account").by("amount").range().build()); + }); + } + + private void createAccounts(BigDecimal amount) { + List vertices = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + Vertex account = new Vertex("account"); + account.property("name", "acc" + i); + account.property("amount", amount); + vertices.add(account); + } + vertexAPI.create(vertices); + } + + private List sum(BigDecimal delta) { + List vertices = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + Vertex account = new Vertex("account"); + account.property("name", "acc" + i); + account.property("amount", delta); + vertices.add(account); + } + Map strategies = ImmutableMap.of( + "amount", UpdateStrategy.SUM); + BatchVertexRequest req = new BatchVertexRequest.Builder() + .vertices(vertices).updatingStrategies(strategies) + .createIfNotExist(true).build(); + List updated = vertexAPI.update(req); + Assert.assertEquals(3, updated.size()); + return updated; + } +} diff --git a/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java b/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java new file mode 100644 index 000000000..a1698e68c --- /dev/null +++ b/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.unit; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Map; + +import org.apache.hugegraph.serializer.direct.util.BytesBuffer; +import org.apache.hugegraph.structure.constant.DataType; +import org.apache.hugegraph.structure.graph.Vertex; +import org.apache.hugegraph.structure.schema.PropertyKey; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.util.JsonUtil; +import org.junit.Test; + +import com.google.common.collect.ImmutableMap; + +public class DecimalDataTypeTest extends BaseUnitTest { + + @Test + public void testDecimalDataType() { + Assert.assertEquals(12, DataType.DECIMAL.code()); + Assert.assertEquals("decimal", DataType.DECIMAL.string()); + Assert.assertEquals(BigDecimal.class, DataType.DECIMAL.clazz()); + Assert.assertTrue(DataType.DECIMAL.isDecimal()); + // Not a Number type: no range index, no sort key, like on the server + Assert.assertFalse(DataType.DECIMAL.isNumber()); + Assert.assertFalse(DataType.DOUBLE.isDecimal()); + } + + @Test + public void testPropertyKeyAsDecimal() { + PropertyKey.Builder builder = new PropertyKey.BuilderImpl("amount", + null); + PropertyKey amount = builder.asDecimal().build(); + Assert.assertEquals(DataType.DECIMAL, amount.dataType()); + String json = JsonUtil.toJson(amount); + Assert.assertContains("\"data_type\":\"DECIMAL\"", json); + } + + @Test + public void testDecimalIsSerializedAsPlainString() { + // Never a JSON number (would be read as a double), never E-notation + Assert.assertEquals("\"1.10\"", + JsonUtil.toJson(new BigDecimal("1.10"))); + Assert.assertEquals("\"1000\"", + JsonUtil.toJson(new BigDecimal("1E+3"))); + Assert.assertEquals("\"0.000000000000000001\"", + JsonUtil.toJson(new BigDecimal("1E-18"))); + Assert.assertEquals("\"-115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935\"", + JsonUtil.toJson(new BigDecimal( + "-115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"))); + + Vertex vertex = new Vertex("account"); + vertex.id("account:1"); + vertex.property("balance", new BigDecimal("12345678901234567890.10")); + String json = serialize(vertex); + Assert.assertContains("\"balance\":\"12345678901234567890.10\"", json); + + // A value read back from the server is the plain string + Vertex copy = deserialize(json, Vertex.class); + Map props = ImmutableMap.of( + "balance", "12345678901234567890.10"); + Assert.assertEquals(props, copy.properties()); + Assert.assertEquals(new BigDecimal("12345678901234567890.10"), + new BigDecimal((String) copy.property("balance"))); + } + + @Test + public void testValueToDecimal() { + DataType type = DataType.DECIMAL; + Assert.assertEquals(new BigDecimal("1.10"), + type.valueToDecimal("1.10")); + Assert.assertEquals(new BigDecimal("1.10"), + type.valueToDecimal(new BigDecimal("1.10"))); + Assert.assertEquals(BigDecimal.valueOf(42L), type.valueToDecimal(42)); + Assert.assertEquals(new BigDecimal("0.1"), type.valueToDecimal(0.1d)); + Assert.assertEquals(new BigDecimal(BigInteger.TEN), + type.valueToDecimal(BigInteger.TEN)); + Assert.assertNull(type.valueToDecimal(true)); + Assert.assertNull(DataType.DOUBLE.valueToDecimal("1.10")); + Assert.assertThrows(IllegalArgumentException.class, () -> { + type.valueToDecimal("1,10"); + }); + + org.apache.hugegraph.serializer.direct.struct.DataType direct = + org.apache.hugegraph.serializer.direct.struct.DataType.DECIMAL; + Assert.assertEquals(12, direct.code()); + Assert.assertTrue(direct.isDecimal()); + Assert.assertFalse(direct.isNumber()); + Assert.assertEquals(new BigDecimal("1.10"), + direct.valueToDecimal("1.10")); + Assert.assertNull(org.apache.hugegraph.serializer.direct.struct + .DataType.DOUBLE.valueToDecimal("1.10")); + } + + @Test + public void testDirectSerializerDecimal() { + // Same layout as the server: unscaled bytes then the scale + BigDecimal value = new BigDecimal("-12345678901234567890.123456789"); + BytesBuffer buffer = BytesBuffer.allocate(64); + buffer.writeProperty(DataType.DECIMAL, value); + buffer.forReadWritten(); + BigInteger unscaled = new BigInteger(buffer.readBytes()); + int scale = buffer.readVInt(); + Assert.assertEquals(value, new BigDecimal(unscaled, scale)); + Assert.assertEquals(0, buffer.remaining()); + + // A string reaches the direct serializer exactly as a decimal would + buffer = BytesBuffer.allocate(16); + buffer.writeProperty(DataType.DECIMAL, "1.10"); + buffer.forReadWritten(); + Assert.assertEquals(new BigDecimal("1.10"), + new BigDecimal(new BigInteger(buffer.readBytes()), + buffer.readVInt())); + } +} diff --git a/hugegraph-client/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-client/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java index 97c32fc1e..4cda8bb31 100644 --- a/hugegraph-client/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-client/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -31,6 +31,7 @@ AuthApiPathTest.class, BatchElementRequestTest.class, PropertyKeyTest.class, + DecimalDataTypeTest.class, IndexLabelTest.class, GraphSpaceAPITest.class, ManagerAPITest.class, diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/schema/GroovySchemaCompatibility.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/schema/GroovySchemaCompatibility.java index 05903ea19..09dc261d9 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/schema/GroovySchemaCompatibility.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/schema/GroovySchemaCompatibility.java @@ -234,6 +234,7 @@ private static String dataType(DataType type) { case DOUBLE: return ".asDouble()"; case FLOAT: return ".asFloat()"; case LONG: return ".asLong()"; + case DECIMAL: return ".asDecimal()"; default: return ".dataType(org.apache.hugegraph.structure.constant." + "DataType." + type.name() + ")"; diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java index cc966be10..0cd191b47 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.loader.util; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -200,6 +201,8 @@ private static Object parseSingleValue(String key, Object rawValue, } } else if (dataType.isUUID()) { return parseUUID(key, value); + } else if (dataType.isDecimal()) { + return parseDecimal(key, value); } else if (dataType.isText()) { if (value instanceof Number) { return value.toString(); @@ -328,6 +331,21 @@ private static Number parseNumber(String key, Object value, } } + private static BigDecimal parseDecimal(String key, Object rawValue) { + BigDecimal decimal; + try { + decimal = DataType.DECIMAL.valueToDecimal(rawValue); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(String.format( + "Failed to convert value(key='%s') '%s'(%s) to Decimal", + key, rawValue, rawValue.getClass()), e); + } + E.checkArgument(decimal != null, + "Failed to convert value(key='%s') '%s'(%s) to Decimal", + key, rawValue, rawValue.getClass()); + return decimal; + } + private static long parseLong(String rawValue) { if (rawValue.startsWith("-")) { return Long.parseLong(rawValue); @@ -403,6 +421,9 @@ private static List split(String key, String rawValue, */ private static boolean checkDataType(String key, Object value, DataType dataType) { + if (dataType.isDecimal()) { + return value instanceof BigDecimal; + } if (value instanceof Number) { return parseNumber(key, value, dataType) != null; } diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java new file mode 100644 index 000000000..bac6a06d9 --- /dev/null +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.loader.test.unit; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.List; + +import org.apache.hugegraph.loader.source.file.FileSource; +import org.apache.hugegraph.loader.source.file.ListFormat; +import org.apache.hugegraph.loader.util.DataTypeUtil; +import org.apache.hugegraph.structure.schema.PropertyKey; +import org.apache.hugegraph.testutil.Assert; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; + +public class DataTypeUtilTest { + + private static final FileSource SOURCE = new FileSource(); + + static { + SOURCE.listFormat(new ListFormat("", "", ",")); + } + + private static PropertyKey decimal(String name) { + return new PropertyKey.BuilderImpl(name, null).asDecimal().build(); + } + + @Test + public void testConvertDecimal() { + PropertyKey amount = decimal("amount"); + Assert.assertEquals(new BigDecimal("1.10"), + DataTypeUtil.convert(" 1.10 ", amount, SOURCE)); + Assert.assertEquals(new BigDecimal("1.10"), + DataTypeUtil.convert(new BigDecimal("1.10"), + amount, SOURCE)); + Assert.assertEquals(BigDecimal.valueOf(42L), + DataTypeUtil.convert(42, amount, SOURCE)); + Assert.assertEquals(BigDecimal.valueOf(-7L), + DataTypeUtil.convert(-7L, amount, SOURCE)); + Assert.assertEquals(new BigDecimal(BigInteger.TEN), + DataTypeUtil.convert(BigInteger.TEN, amount, + SOURCE)); + // A double keeps its shortest round-trip representation + Assert.assertEquals(new BigDecimal("0.1"), + DataTypeUtil.convert(0.1d, amount, SOURCE)); + // 78 digits (uint256 max) stay exact + String uint256 = "115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"; + Assert.assertEquals(new BigDecimal(uint256), + DataTypeUtil.convert(uint256, amount, SOURCE)); + + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataTypeUtil.convert("1,10", amount, SOURCE); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataTypeUtil.convert(true, amount, SOURCE); + }); + } + + @Test + public void testConvertDecimalList() { + PropertyKey amounts = new PropertyKey.BuilderImpl("amounts", null) + .asDecimal().valueList().build(); + Object values = DataTypeUtil.convert("1.10,2", amounts, SOURCE); + Assert.assertEquals(ImmutableList.of(new BigDecimal("1.10"), + new BigDecimal("2")), values); + // A parsed JSON list is accepted only when its elements are decimals + List parsed = ImmutableList.of(new BigDecimal("3.30")); + Assert.assertEquals(parsed, + DataTypeUtil.convert(parsed, amounts, SOURCE)); + Assert.assertThrows(IllegalStateException.class, () -> { + DataTypeUtil.convert(ImmutableList.of(3.3d), amounts, SOURCE); + }); + } +} diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java index b9d453cfe..ae66e5ccb 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java @@ -24,6 +24,7 @@ @Suite.SuiteClasses({ LineTest.class, DateUtilTest.class, + DataTypeUtilTest.class, MappingConverterTest.class, LoadProgressTest.class, RangesTimerTest.class diff --git a/hugegraph-spark-connector/src/main/java/org/apache/hugegraph/spark/connector/utils/DataTypeUtils.java b/hugegraph-spark-connector/src/main/java/org/apache/hugegraph/spark/connector/utils/DataTypeUtils.java index 317e7568f..ac0d30fc3 100644 --- a/hugegraph-spark-connector/src/main/java/org/apache/hugegraph/spark/connector/utils/DataTypeUtils.java +++ b/hugegraph-spark-connector/src/main/java/org/apache/hugegraph/spark/connector/utils/DataTypeUtils.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.spark.connector.utils; +import java.math.BigDecimal; import java.util.Date; import java.util.Set; import java.util.UUID; @@ -74,6 +75,8 @@ private static Object parseSingleValue(String key, Object rawValue, DataType dat return parseDate(key, value, dateFormat, timeZone); } else if (dataType.isUUID()) { return parseUUID(key, value); + } else if (dataType.isDecimal()) { + return parseDecimal(key, value); } else if (dataType.isText()) { if (!(rawValue instanceof String)) { value = rawValue.toString(); @@ -98,6 +101,21 @@ public static long parseNumber(String key, Object rawValue) { rawValue, rawValue.getClass().getName())); } + public static BigDecimal parseDecimal(String key, Object rawValue) { + BigDecimal decimal; + try { + decimal = DataType.DECIMAL.valueToDecimal(rawValue); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(String.format( + "The value(key='%s') '%s'(%s) can't be casted to Decimal", + key, rawValue, rawValue.getClass()), e); + } + E.checkArgument(decimal != null, + "The value(key='%s') '%s'(%s) can't be casted to Decimal", + key, rawValue, rawValue.getClass()); + return decimal; + } + public static UUID parseUUID(String key, Object rawValue) { if (rawValue instanceof UUID) { return (UUID) rawValue; From a57545f308291678b97d64674c762d7f64036800 Mon Sep 17 00:00:00 2001 From: Sebastian Gruza Date: Mon, 21 Sep 2026 21:39:44 +0000 Subject: [PATCH 2/8] fix(client): send BigDecimal as a plain JSON number, apply decimal bounds, keep loader JSON decimals exact Review round 1 of the DECIMAL companion: - A BigDecimal is written as a plain JSON number (every digit, never E-notation) instead of a string. Numeric keys keep accepting it (the server narrows any Number), so `vertex.property("weight", new BigDecimal("1.5"))` on a DOUBLE key works against every server, and a server that reads fractions as BigDecimal (apache/hugegraph#3209) stores a DECIMAL value exactly. Covered by `VertexApiTest.testCreateWithBigDecimalOnDoubleKey` (runs on the CI server) and `DecimalDataTypeTest`. - Both client `valueToDecimal` copies apply the server's bounds (128 significant digits, scale 128 either way), so the HBase direct path cannot store a value such as 1E+999999999; unit-tested on both copies and on `BytesBuffer.writeProperty`. - The loader's JSON line parser reads fractions as BigDecimal (`USE_BIG_DECIMAL_FOR_FLOATS`), so a decimal column keeps digits a double would drop; collection elements of another type (a JSON integer in a decimal list) are converted one by one. `DataTypeUtilTest` covers a 39-digit JSON value, a mixed list and DOUBLE narrowing. --- .../apache/hugegraph/client/RestClient.java | 2 +- .../serializer/BigDecimalSerializer.java | 12 ++-- .../serializer/direct/struct/DataType.java | 42 ++++++++--- .../structure/constant/DataType.java | 42 ++++++++--- .../org/apache/hugegraph/util/JsonUtil.java | 6 +- .../apache/hugegraph/api/VertexApiTest.java | 21 ++++++ .../hugegraph/unit/DecimalDataTypeTest.java | 70 +++++++++++++------ .../hugegraph/loader/util/DataTypeUtil.java | 21 ++++-- .../hugegraph/loader/util/JsonUtil.java | 10 +++ .../loader/test/unit/DataTypeUtilTest.java | 38 +++++++++- 10 files changed, 211 insertions(+), 53 deletions(-) diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/client/RestClient.java b/hugegraph-client/src/main/java/org/apache/hugegraph/client/RestClient.java index 7c0f7bbf0..730eb3349 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/client/RestClient.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/client/RestClient.java @@ -53,7 +53,7 @@ public class RestClient extends AbstractRestClient { module.addDeserializer(Path.class, new PathDeserializer()); RestResult.registerModule(module); - // Request bodies: a decimal goes as a plain string, see JsonUtil + // Request bodies: a BigDecimal goes as a plain number, see JsonUtil SimpleModule decimals = new SimpleModule(); decimals.addSerializer(BigDecimal.class, new BigDecimalSerializer()); JsonUtilCommon.registerModule(decimals); diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java index cd30e933f..d4838e963 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java @@ -25,10 +25,12 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer; /** - * Write a BigDecimal as a plain decimal string ("1.10", never "1.1E+2"). - * A JSON number would be parsed as a double on the server side and lose - * both precision and trailing zeros; a string reaches a DECIMAL property - * key exactly, which is the point of that data type. + * Write a BigDecimal as a plain JSON number ("1.10", never "1.1E+2"). + * Jackson's default is the scientific form of BigDecimal.toString(); the + * plain form carries every digit, so a server that reads fractions as + * BigDecimal (apache/hugegraph#3209) stores a DECIMAL value exactly, and a + * server that reads them as double behaves as it always did. The value stays + * a number, so numeric keys (DOUBLE, FLOAT, LONG, INT) accept it too. */ public class BigDecimalSerializer extends StdSerializer { @@ -41,6 +43,6 @@ public BigDecimalSerializer() { @Override public void serialize(BigDecimal value, JsonGenerator generator, SerializerProvider provider) throws IOException { - generator.writeString(value.toPlainString()); + generator.writeNumber(value.toPlainString()); } } diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/struct/DataType.java b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/struct/DataType.java index 216a5567a..286628d07 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/struct/DataType.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/struct/DataType.java @@ -200,10 +200,20 @@ public UUID valueToUUID(V value) { return null; } + /* + * Bounds for a DECIMAL value, the same as the server applies: at most + * DECIMAL_MAX_PRECISION significant digits and an absolute scale of at + * most DECIMAL_MAX_SCALE. A value such as "1E+999999999" costs a few bytes + * to store and a billion characters on every read, and the direct loaders + * write bytes into storage without the server's check. + */ + public static final int DECIMAL_MAX_PRECISION = 128; + public static final int DECIMAL_MAX_SCALE = 128; + /** * Convert a value to BigDecimal the same way the server does: BigDecimal * as is, integral numbers exactly, any other Number and a decimal string - * through their decimal representation. + * through their decimal representation, then checked against the bounds. * * @return the BigDecimal, or null if the value can't be a decimal */ @@ -211,22 +221,38 @@ public BigDecimal valueToDecimal(V value) { if (!this.isDecimal()) { return null; } + BigDecimal decimal; if (value instanceof BigDecimal) { - return (BigDecimal) value; + decimal = (BigDecimal) value; } else if (value instanceof BigInteger) { - return new BigDecimal((BigInteger) value); + decimal = new BigDecimal((BigInteger) value); } else if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long) { - return BigDecimal.valueOf(((Number) value).longValue()); + decimal = BigDecimal.valueOf(((Number) value).longValue()); } else if (!(value instanceof Number) && !(value instanceof String)) { return null; + } else { + try { + decimal = new BigDecimal(value.toString().trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "Can't read '%s' as decimal", value)); + } } - try { - return new BigDecimal(value.toString().trim()); - } catch (NumberFormatException e) { + return checkDecimalBounds(decimal); + } + + public static BigDecimal checkDecimalBounds(BigDecimal decimal) { + int scale = Math.abs(decimal.scale()); + int precision = decimal.precision(); + if (precision > DECIMAL_MAX_PRECISION || scale > DECIMAL_MAX_SCALE) { throw new IllegalArgumentException(String.format( - "Can't read '%s' as decimal", value)); + "Decimal value out of bounds: precision %d, scale %d " + + "(at most %d significant digits and a scale of at most " + + "%d in either direction)", precision, decimal.scale(), + DECIMAL_MAX_PRECISION, DECIMAL_MAX_SCALE)); } + return decimal; } public static DataType fromClass(Class clazz) { diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java b/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java index 57b89b994..42180d9a4 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java @@ -82,10 +82,20 @@ public boolean isDecimal() { return this == DataType.DECIMAL; } + /* + * Bounds for a DECIMAL value, the same as the server applies: at most + * DECIMAL_MAX_PRECISION significant digits and an absolute scale of at + * most DECIMAL_MAX_SCALE. A value such as "1E+999999999" costs a few bytes + * to store and a billion characters on every read, and the direct loaders + * write bytes into storage without the server's check. + */ + public static final int DECIMAL_MAX_PRECISION = 128; + public static final int DECIMAL_MAX_SCALE = 128; + /** * Convert a value to BigDecimal the same way the server does: BigDecimal * as is, integral numbers exactly, any other Number and a decimal string - * through their decimal representation. + * through their decimal representation, then checked against the bounds. * * @return the BigDecimal, or null if the value can't be a decimal * @throws IllegalArgumentException if the string is not a decimal number @@ -94,22 +104,38 @@ public BigDecimal valueToDecimal(V value) { if (!this.isDecimal()) { return null; } + BigDecimal decimal; if (value instanceof BigDecimal) { - return (BigDecimal) value; + decimal = (BigDecimal) value; } else if (value instanceof BigInteger) { - return new BigDecimal((BigInteger) value); + decimal = new BigDecimal((BigInteger) value); } else if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long) { - return BigDecimal.valueOf(((Number) value).longValue()); + decimal = BigDecimal.valueOf(((Number) value).longValue()); } else if (!(value instanceof Number) && !(value instanceof String)) { return null; + } else { + try { + decimal = new BigDecimal(value.toString().trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "Can't read '%s' as decimal", value)); + } } - try { - return new BigDecimal(value.toString().trim()); - } catch (NumberFormatException e) { + return checkDecimalBounds(decimal); + } + + public static BigDecimal checkDecimalBounds(BigDecimal decimal) { + int scale = Math.abs(decimal.scale()); + int precision = decimal.precision(); + if (precision > DECIMAL_MAX_PRECISION || scale > DECIMAL_MAX_SCALE) { throw new IllegalArgumentException(String.format( - "Can't read '%s' as decimal", value)); + "Decimal value out of bounds: precision %d, scale %d " + + "(at most %d significant digits and a scale of at most " + + "%d in either direction)", precision, decimal.scale(), + DECIMAL_MAX_PRECISION, DECIMAL_MAX_SCALE)); } + return decimal; } public boolean isBoolean() { diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/util/JsonUtil.java b/hugegraph-client/src/main/java/org/apache/hugegraph/util/JsonUtil.java index c34acbcc2..a2c293fbc 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/util/JsonUtil.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/util/JsonUtil.java @@ -35,9 +35,9 @@ public final class JsonUtil { static { /* - * A DECIMAL property value travels as a plain string ("1.10"): a - * JSON number is read as a double on the other side. The same - * serializer is registered for request bodies in RestClient. + * A BigDecimal travels as a plain JSON number ("1.10", not "1.1E+2") + * so that a DECIMAL key receives every digit. The same serializer is + * registered for request bodies in RestClient. */ SimpleModule module = new SimpleModule(); module.addSerializer(BigDecimal.class, new BigDecimalSerializer()); diff --git a/hugegraph-client/src/test/java/org/apache/hugegraph/api/VertexApiTest.java b/hugegraph-client/src/test/java/org/apache/hugegraph/api/VertexApiTest.java index 49530a5e9..d79d0af3d 100644 --- a/hugegraph-client/src/test/java/org/apache/hugegraph/api/VertexApiTest.java +++ b/hugegraph-client/src/test/java/org/apache/hugegraph/api/VertexApiTest.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.api; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -61,6 +62,26 @@ public void teardown() { vertexAPI.list(-1).results().forEach(v -> vertexAPI.delete(v.id())); } + @Test + public void testCreateWithBigDecimalOnDoubleKey() { + // A BigDecimal sent to a DOUBLE key is a JSON number, so the server + // narrows it to a double as it does for any numeric literal + SchemaManager schema = schema(); + schema.propertyKey("weight").asDouble().ifNotExist().create(); + schema.vertexLabel("scale").properties("name", "weight") + .primaryKeys("name").nullableKeys("weight") + .ifNotExist().create(); + + Vertex vertex = new Vertex("scale"); + vertex.property("name", "kg"); + vertex.property("weight", new BigDecimal("1.5")); + vertex = vertexAPI.create(vertex); + Assert.assertEquals(1.5, vertex.property("weight")); + + vertex = vertexAPI.get(vertex.id()); + Assert.assertEquals(1.5, vertex.property("weight")); + } + @Test public void testCreate() { Vertex vertex = new Vertex("person"); diff --git a/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java b/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java index a1698e68c..c50f92a92 100644 --- a/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java +++ b/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java @@ -19,8 +19,6 @@ import java.math.BigDecimal; import java.math.BigInteger; -import java.util.Map; - import org.apache.hugegraph.serializer.direct.util.BytesBuffer; import org.apache.hugegraph.structure.constant.DataType; import org.apache.hugegraph.structure.graph.Vertex; @@ -29,8 +27,6 @@ import org.apache.hugegraph.util.JsonUtil; import org.junit.Test; -import com.google.common.collect.ImmutableMap; - public class DecimalDataTypeTest extends BaseUnitTest { @Test @@ -55,35 +51,65 @@ public void testPropertyKeyAsDecimal() { } @Test - public void testDecimalIsSerializedAsPlainString() { - // Never a JSON number (would be read as a double), never E-notation - Assert.assertEquals("\"1.10\"", - JsonUtil.toJson(new BigDecimal("1.10"))); - Assert.assertEquals("\"1000\"", - JsonUtil.toJson(new BigDecimal("1E+3"))); - Assert.assertEquals("\"0.000000000000000001\"", + public void testDecimalIsSerializedAsPlainNumber() { + // A plain JSON number: every digit, never E-notation, never a string + Assert.assertEquals("1.10", JsonUtil.toJson(new BigDecimal("1.10"))); + Assert.assertEquals("1000", JsonUtil.toJson(new BigDecimal("1E+3"))); + Assert.assertEquals("0.000000000000000001", JsonUtil.toJson(new BigDecimal("1E-18"))); - Assert.assertEquals("\"-115792089237316195423570985008687907853" + - "269984665640564039457584007913129639935\"", - JsonUtil.toJson(new BigDecimal( - "-115792089237316195423570985008687907853" + - "269984665640564039457584007913129639935"))); + String uint256Max = "115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"; + Assert.assertEquals("-" + uint256Max, + JsonUtil.toJson(new BigDecimal("-" + uint256Max))); Vertex vertex = new Vertex("account"); vertex.id("account:1"); vertex.property("balance", new BigDecimal("12345678901234567890.10")); String json = serialize(vertex); - Assert.assertContains("\"balance\":\"12345678901234567890.10\"", json); + Assert.assertContains("\"balance\":12345678901234567890.10", json); - // A value read back from the server is the plain string - Vertex copy = deserialize(json, Vertex.class); - Map props = ImmutableMap.of( - "balance", "12345678901234567890.10"); - Assert.assertEquals(props, copy.properties()); + // A DECIMAL value read back from the server is a plain string + Vertex copy = deserialize("{\"id\":\"account:1\",\"label\":" + + "\"account\",\"type\":\"vertex\"," + + "\"properties\":{\"balance\":" + + "\"12345678901234567890.10\"}}", + Vertex.class); Assert.assertEquals(new BigDecimal("12345678901234567890.10"), new BigDecimal((String) copy.property("balance"))); } + @Test + public void testDecimalBounds() { + DataType type = DataType.DECIMAL; + // 128 significant digits and scale 128 are the server's limits + String digits128 = new String(new char[128]).replace("\0", "7"); + Assert.assertEquals(new BigDecimal(digits128), + type.valueToDecimal(digits128)); + Assert.assertEquals(new BigDecimal("1E-128"), + type.valueToDecimal("1E-128")); + Assert.assertEquals(new BigDecimal("1E+128"), + type.valueToDecimal("1E+128")); + Assert.assertThrows(IllegalArgumentException.class, () -> { + type.valueToDecimal(digits128 + "7"); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + type.valueToDecimal("1E-129"); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + type.valueToDecimal(new BigDecimal("1E+999999999")); + }); + // the direct-serializer copy applies the same bounds before writing + org.apache.hugegraph.serializer.direct.struct.DataType direct = + org.apache.hugegraph.serializer.direct.struct.DataType.DECIMAL; + Assert.assertThrows(IllegalArgumentException.class, () -> { + direct.valueToDecimal("1E+999999999"); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + BytesBuffer.allocate(16).writeProperty(DataType.DECIMAL, + new BigDecimal("1E+999999999")); + }); + } + @Test public void testValueToDecimal() { DataType type = DataType.DECIMAL; diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java index 0cd191b47..c6b21f2d1 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java @@ -226,10 +226,23 @@ private static Object parseMultiValues(String key, Object values, DataType dataType, Cardinality cardinality, InputSource source) { - // JSON file should not parse again - if (values instanceof Collection && - checkCollectionDataType(key, (Collection) values, dataType)) { - return values; + if (values instanceof Collection) { + // JSON file should not parse again + if (checkCollectionDataType(key, (Collection) values, dataType)) { + return values; + } + // Elements of another type (e.g. a JSON integer literal in a + // decimal list) are converted one by one + Collection converted = cardinality == Cardinality.LIST ? + InsertionOrderUtil.newList() : + InsertionOrderUtil.newSet(); + for (Object value : (Collection) values) { + converted.add(parseSingleValue(key, value, dataType, source)); + } + E.checkArgument(checkCollectionDataType(key, converted, dataType), + "Not all collection elems %s match with data type %s", + converted, dataType); + return converted; } E.checkState(values instanceof String, diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/JsonUtil.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/JsonUtil.java index 7b23fc48c..ed4c4a587 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/JsonUtil.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/JsonUtil.java @@ -39,6 +39,7 @@ import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; @@ -47,6 +48,15 @@ public final class JsonUtil { private static final Logger LOG = Log.logger(JsonUtil.class); private static final ObjectMapper MAPPER = new ObjectMapper(); + static { + /* + * Read JSON fraction literals as BigDecimal: a DECIMAL column keeps + * every digit (a double would keep 17), the other numeric types are + * narrowed by DataTypeUtil as before. + */ + MAPPER.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + } + static { SimpleModule module = new SimpleModule(); module.addDeserializer(InputSource.class, new InputSourceDeser()); diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java index bac6a06d9..5750317a6 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java @@ -20,10 +20,12 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.util.List; +import java.util.Map; import org.apache.hugegraph.loader.source.file.FileSource; import org.apache.hugegraph.loader.source.file.ListFormat; import org.apache.hugegraph.loader.util.DataTypeUtil; +import org.apache.hugegraph.loader.util.JsonUtil; import org.apache.hugegraph.structure.schema.PropertyKey; import org.apache.hugegraph.testutil.Assert; import org.junit.Test; @@ -85,8 +87,40 @@ public void testConvertDecimalList() { List parsed = ImmutableList.of(new BigDecimal("3.30")); Assert.assertEquals(parsed, DataTypeUtil.convert(parsed, amounts, SOURCE)); - Assert.assertThrows(IllegalStateException.class, () -> { - DataTypeUtil.convert(ImmutableList.of(3.3d), amounts, SOURCE); + // elements of another type are converted one by one + Assert.assertEquals(ImmutableList.of(new BigDecimal("3.3")), + DataTypeUtil.convert(ImmutableList.of(3.3d), + amounts, SOURCE)); + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataTypeUtil.convert(ImmutableList.of("x"), amounts, SOURCE); }); } + + @Test + public void testConvertDecimalFromJsonLine() { + // The JSON line parser reads fractions as BigDecimal, so a value + // with more digits than a double holds arrives intact, and a list + // column comes as a list of decimals + PropertyKey amount = decimal("amount"); + PropertyKey amounts = new PropertyKey.BuilderImpl("amounts", null) + .asDecimal().valueList().build(); + PropertyKey weight = new PropertyKey.BuilderImpl("weight", null) + .asDouble().build(); + Map line = JsonUtil.convertMap( + "{\"amount\": 12345678901234567890.123456789012345678," + + " \"amounts\": [1.10, 2, 3E-18], \"weight\": 2.5}", + String.class, Object.class); + Assert.assertEquals( + new BigDecimal("12345678901234567890.123456789012345678"), + DataTypeUtil.convert(line.get("amount"), amount, SOURCE)); + Assert.assertEquals(ImmutableList.of(new BigDecimal("1.10"), + new BigDecimal("2"), + new BigDecimal("3E-18")), + DataTypeUtil.convert(line.get("amounts"), amounts, + SOURCE)); + // other numeric keys are narrowed as before + Assert.assertEquals(2.5d, + DataTypeUtil.convert(line.get("weight"), weight, + SOURCE)); + } } From 56fcad0e8b708d0c882f418c6f809064c121a281 Mon Sep 17 00:00:00 2001 From: Sebastian Gruza Date: Tue, 22 Sep 2026 16:59:00 +0000 Subject: [PATCH 3/8] fix(loader): keep the TEXT-key string of JSON fractions; document the DECIMAL wire format Review round 2 of the DECIMAL companion: - The loader parses JSON fractions as BigDecimal since a57545f3; a fraction loaded into a TEXT key printed differently ("1.50" instead of "1.5"), which would change primary-key ids on a reload. The TEXT branch now formats a BigDecimal through Double.toString, the string the Double path produced; DataTypeUtilTest pins 1.50, 1e-7, 12345678901.0 and an integer. - The public DataType.DECIMAL comment describes the actual wire format: sent as a plain JSON number, returned by the server as a plain decimal string. --- .../hugegraph/structure/constant/DataType.java | 4 +++- .../apache/hugegraph/loader/util/DataTypeUtil.java | 6 ++++++ .../loader/test/unit/DataTypeUtilTest.java | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java b/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java index 42180d9a4..efa0e83cf 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java @@ -38,7 +38,9 @@ public enum DataType { UUID(11, "uuid", UUID.class), /* * Arbitrary-precision decimal (java.math.BigDecimal), stored exactly by - * the server; sent and received as a plain decimal string in JSON + * the server. On the wire: sent as a plain JSON number + * (BigDecimal.toPlainString(), every digit), returned by the server as a + * plain decimal string; new BigDecimal(String) restores it exactly */ DECIMAL(12, "decimal", BigDecimal.class); diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java index c6b21f2d1..843e73928 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java @@ -204,6 +204,12 @@ private static Object parseSingleValue(String key, Object rawValue, } else if (dataType.isDecimal()) { return parseDecimal(key, value); } else if (dataType.isText()) { + if (value instanceof BigDecimal) { + // JSON fractions are parsed as BigDecimal (see JsonUtil); + // a TEXT key keeps the string the double path produced + // ("1.5", not "1.50"), so existing ids do not change + return Double.toString(((BigDecimal) value).doubleValue()); + } if (value instanceof Number) { return value.toString(); } diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java index 5750317a6..f0b4b310d 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java @@ -122,5 +122,19 @@ public void testConvertDecimalFromJsonLine() { Assert.assertEquals(2.5d, DataTypeUtil.convert(line.get("weight"), weight, SOURCE)); + // a TEXT key gets the same string as before the BigDecimal parsing + PropertyKey label = new PropertyKey.BuilderImpl("label", null) + .asText().build(); + Map texts = JsonUtil.convertMap( + "{\"a\": 1.50, \"b\": 1e-7, \"c\": 12345678901.0, \"d\": 7}", + String.class, Object.class); + Assert.assertEquals("1.5", + DataTypeUtil.convert(texts.get("a"), label, SOURCE)); + Assert.assertEquals("1.0E-7", + DataTypeUtil.convert(texts.get("b"), label, SOURCE)); + Assert.assertEquals("1.2345678901E10", + DataTypeUtil.convert(texts.get("c"), label, SOURCE)); + Assert.assertEquals("7", + DataTypeUtil.convert(texts.get("d"), label, SOURCE)); } } From 268af65c8e101f82a8e396af6570a53e00ba4173 Mon Sep 17 00:00:00 2001 From: Sebastian Gruza Date: Wed, 30 Sep 2026 06:54:44 +0000 Subject: [PATCH 4/8] fix(loader): format a BigDecimal for a TEXT key through double only when it came from the JSON parser Review round 3: JDBC sources hand BigDecimal for every DECIMAL/NUMERIC column, and the round-2 TEXT branch formatted those through double too (12.50 -> 12.5, long values lossy), changing primary-key ids on reload. The double formatting now applies only to sources parsed as JSON (FileSource/HDFSSource and KafkaSource with format JSON); any other BigDecimal keeps its own text. DataTypeUtilTest pins the JDBC case. --- .../hugegraph/loader/util/DataTypeUtil.java | 17 +++++++-- .../loader/test/unit/DataTypeUtilTest.java | 36 ++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java index 843e73928..e8a1e1053 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java @@ -33,6 +33,7 @@ import org.apache.hugegraph.loader.constant.Constants; import org.apache.hugegraph.loader.source.AbstractSource; import org.apache.hugegraph.loader.source.InputSource; +import org.apache.hugegraph.loader.source.file.FileFormat; import org.apache.hugegraph.loader.source.file.FileSource; import org.apache.hugegraph.loader.source.file.ListFormat; import org.apache.hugegraph.loader.source.hdfs.HDFSSource; @@ -204,10 +205,12 @@ private static Object parseSingleValue(String key, Object rawValue, } else if (dataType.isDecimal()) { return parseDecimal(key, value); } else if (dataType.isText()) { - if (value instanceof BigDecimal) { + if (value instanceof BigDecimal && fromJsonParser(source)) { // JSON fractions are parsed as BigDecimal (see JsonUtil); // a TEXT key keeps the string the double path produced - // ("1.5", not "1.50"), so existing ids do not change + // ("1.5", not "1.50"), so existing ids do not change. + // A BigDecimal from any other source (JDBC DECIMAL/NUMERIC) + // keeps its own text, as before return Double.toString(((BigDecimal) value).doubleValue()); } if (value instanceof Number) { @@ -350,6 +353,16 @@ private static Number parseNumber(String key, Object value, } } + private static boolean fromJsonParser(InputSource source) { + if (source instanceof FileSource) { + return ((FileSource) source).format() == FileFormat.JSON; + } + if (source instanceof KafkaSource) { + return ((KafkaSource) source).getFormat() == FileFormat.JSON; + } + return false; + } + private static BigDecimal parseDecimal(String key, Object rawValue) { BigDecimal decimal; try { diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java index f0b4b310d..e31d6cd54 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java @@ -22,8 +22,10 @@ import java.util.List; import java.util.Map; +import org.apache.hugegraph.loader.source.file.FileFormat; import org.apache.hugegraph.loader.source.file.FileSource; import org.apache.hugegraph.loader.source.file.ListFormat; +import org.apache.hugegraph.loader.source.jdbc.JDBCSource; import org.apache.hugegraph.loader.util.DataTypeUtil; import org.apache.hugegraph.loader.util.JsonUtil; import org.apache.hugegraph.structure.schema.PropertyKey; @@ -35,9 +37,12 @@ public class DataTypeUtilTest { private static final FileSource SOURCE = new FileSource(); + private static final FileSource JSON_SOURCE = new FileSource(); + private static final JDBCSource JDBC_SOURCE = new JDBCSource(); static { SOURCE.listFormat(new ListFormat("", "", ",")); + JSON_SOURCE.format(FileFormat.JSON); } private static PropertyKey decimal(String name) { @@ -129,12 +134,35 @@ public void testConvertDecimalFromJsonLine() { "{\"a\": 1.50, \"b\": 1e-7, \"c\": 12345678901.0, \"d\": 7}", String.class, Object.class); Assert.assertEquals("1.5", - DataTypeUtil.convert(texts.get("a"), label, SOURCE)); + DataTypeUtil.convert(texts.get("a"), label, JSON_SOURCE)); Assert.assertEquals("1.0E-7", - DataTypeUtil.convert(texts.get("b"), label, SOURCE)); + DataTypeUtil.convert(texts.get("b"), label, JSON_SOURCE)); Assert.assertEquals("1.2345678901E10", - DataTypeUtil.convert(texts.get("c"), label, SOURCE)); + DataTypeUtil.convert(texts.get("c"), label, JSON_SOURCE)); Assert.assertEquals("7", - DataTypeUtil.convert(texts.get("d"), label, SOURCE)); + DataTypeUtil.convert(texts.get("d"), label, JSON_SOURCE)); + } + + @Test + public void testConvertJdbcDecimalToTextKeepsItsText() { + // A BigDecimal from JDBC (DECIMAL/NUMERIC, Oracle NUMBER) is not a + // parsed JSON fraction: a TEXT key stores its own text, as before + PropertyKey label = new PropertyKey.BuilderImpl("label", null) + .asText().build(); + Assert.assertEquals("12.50", + DataTypeUtil.convert(new BigDecimal("12.50"), label, + JDBC_SOURCE)); + Assert.assertEquals("12345", + DataTypeUtil.convert(new BigDecimal("12345"), label, + JDBC_SOURCE)); + Assert.assertEquals("12345678901234567890.12", + DataTypeUtil.convert( + new BigDecimal("12345678901234567890.12"), + label, JDBC_SOURCE)); + // and a DECIMAL key keeps it exact from either source + Assert.assertEquals(new BigDecimal("12345678901234567890.12"), + DataTypeUtil.convert( + new BigDecimal("12345678901234567890.12"), + decimal("amount"), JDBC_SOURCE)); } } From 4ca9ce2e541f8304ca03ffd6bd474cc83b32374a Mon Sep 17 00:00:00 2001 From: Sebastian Gruza Date: Wed, 30 Sep 2026 12:52:39 +0000 Subject: [PATCH 5/8] fix(client,loader): bound the plain DECIMAL form, compare scales directly, check list elements, match numeric null values by value - BigDecimalSerializer writes the plain form only within the DECIMAL bounds (128 digits, scale within +-128) and the scientific form beyond them, so a value such as 1E+999999999 is not expanded on the client before the server rejects it. - checkDecimalBounds compares the scale to both limits directly in both DataType copies: Math.abs(Integer.MIN_VALUE) stays negative. - Loader: a BigDecimal list element counts as already typed only within the bounds; otherwise it goes through parseDecimal and is rejected like the scalar. - Loader: numeric null_values match a field by value (BigDecimal.equals is scale sensitive; both sides are BigDecimal since the mapper change). - DecimalPropertyApiTest skips only on the unsupported-enum response of a server without the type and rethrows anything else. Co-Authored-By: Claude Opus 4.8 --- .../serializer/BigDecimalSerializer.java | 19 ++++++- .../serializer/direct/struct/DataType.java | 6 ++- .../structure/constant/DataType.java | 6 ++- .../hugegraph/api/DecimalPropertyApiTest.java | 12 ++++- .../hugegraph/unit/DecimalDataTypeTest.java | 31 +++++++++++ .../loader/builder/ElementBuilder.java | 39 +++++++++++++- .../hugegraph/loader/util/DataTypeUtil.java | 13 ++++- .../loader/test/unit/DataTypeUtilTest.java | 9 ++++ .../loader/test/unit/NullValuesTest.java | 53 +++++++++++++++++++ .../loader/test/unit/UnitTestSuite.java | 1 + 10 files changed, 180 insertions(+), 9 deletions(-) create mode 100644 hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java index d4838e963..5db1f2430 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.math.BigDecimal; +import org.apache.hugegraph.structure.constant.DataType; + import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.ser.std.StdSerializer; @@ -43,6 +45,21 @@ public BigDecimalSerializer() { @Override public void serialize(BigDecimal value, JsonGenerator generator, SerializerProvider provider) throws IOException { - generator.writeNumber(value.toPlainString()); + generator.writeNumber(exactString(value)); + } + + /** + * The plain form while the value is within the DECIMAL bounds the server + * accepts, else the scientific form: still exact and still a JSON number, + * but a value such as 1E+999999999 is not expanded into a billion + * characters on the client before the server rejects it. + */ + public static String exactString(BigDecimal value) { + int scale = value.scale(); + if (value.precision() <= DataType.DECIMAL_MAX_PRECISION && + scale >= -DataType.DECIMAL_MAX_SCALE && scale <= DataType.DECIMAL_MAX_SCALE) { + return value.toPlainString(); + } + return value.toString(); } } diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/struct/DataType.java b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/struct/DataType.java index 286628d07..588f40c25 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/struct/DataType.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/struct/DataType.java @@ -243,9 +243,11 @@ public BigDecimal valueToDecimal(V value) { } public static BigDecimal checkDecimalBounds(BigDecimal decimal) { - int scale = Math.abs(decimal.scale()); + // Compare the scale directly: Math.abs(Integer.MIN_VALUE) stays negative + int scale = decimal.scale(); int precision = decimal.precision(); - if (precision > DECIMAL_MAX_PRECISION || scale > DECIMAL_MAX_SCALE) { + if (precision > DECIMAL_MAX_PRECISION || + scale < -DECIMAL_MAX_SCALE || scale > DECIMAL_MAX_SCALE) { throw new IllegalArgumentException(String.format( "Decimal value out of bounds: precision %d, scale %d " + "(at most %d significant digits and a scale of at most " + diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java b/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java index efa0e83cf..12b8790d7 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/structure/constant/DataType.java @@ -128,9 +128,11 @@ public BigDecimal valueToDecimal(V value) { } public static BigDecimal checkDecimalBounds(BigDecimal decimal) { - int scale = Math.abs(decimal.scale()); + // Compare the scale directly: Math.abs(Integer.MIN_VALUE) stays negative + int scale = decimal.scale(); int precision = decimal.precision(); - if (precision > DECIMAL_MAX_PRECISION || scale > DECIMAL_MAX_SCALE) { + if (precision > DECIMAL_MAX_PRECISION || + scale < -DECIMAL_MAX_SCALE || scale > DECIMAL_MAX_SCALE) { throw new IllegalArgumentException(String.format( "Decimal value out of bounds: precision %d, scale %d " + "(at most %d significant digits and a scale of at most " + diff --git a/hugegraph-client/src/test/java/org/apache/hugegraph/api/DecimalPropertyApiTest.java b/hugegraph-client/src/test/java/org/apache/hugegraph/api/DecimalPropertyApiTest.java index cfcd30b27..c62abd525 100644 --- a/hugegraph-client/src/test/java/org/apache/hugegraph/api/DecimalPropertyApiTest.java +++ b/hugegraph-client/src/test/java/org/apache/hugegraph/api/DecimalPropertyApiTest.java @@ -58,8 +58,16 @@ public static void prepareSchema() { try { propertyKeyAPI.create(amount); } catch (ServerException e) { - Assume.assumeTrue("The server has no DECIMAL data type: " + - e.getMessage(), false); + // A server without the type rejects the enum value while + // deserializing the request; any other failure is a real one + String message = String.valueOf(e.getMessage()); + boolean noDecimalType = message.contains("DataType") && + message.contains("\"DECIMAL\"") && + message.contains("not one of the values accepted"); + if (!noDecimalType) { + throw e; + } + Assume.assumeTrue("The server has no DECIMAL data type: " + message, false); } propertyKeyAPI.create(new PropertyKey.BuilderImpl("name", null) .asText().build()); diff --git a/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java b/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java index c50f92a92..2874e36d6 100644 --- a/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java +++ b/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java @@ -19,6 +19,7 @@ import java.math.BigDecimal; import java.math.BigInteger; +import org.apache.hugegraph.serializer.BigDecimalSerializer; import org.apache.hugegraph.serializer.direct.util.BytesBuffer; import org.apache.hugegraph.structure.constant.DataType; import org.apache.hugegraph.structure.graph.Vertex; @@ -27,6 +28,8 @@ import org.apache.hugegraph.util.JsonUtil; import org.junit.Test; +import com.google.common.collect.ImmutableMap; + public class DecimalDataTypeTest extends BaseUnitTest { @Test @@ -108,6 +111,34 @@ public void testDecimalBounds() { BytesBuffer.allocate(16).writeProperty(DataType.DECIMAL, new BigDecimal("1E+999999999")); }); + // Math.abs(Integer.MIN_VALUE) stays negative: the scale is compared directly + BigDecimal minScale = new BigDecimal(BigInteger.ONE, Integer.MIN_VALUE); + BigDecimal maxScale = new BigDecimal(BigInteger.ONE, Integer.MAX_VALUE); + for (BigDecimal extreme : new BigDecimal[]{minScale, maxScale}) { + Assert.assertThrows(IllegalArgumentException.class, () -> { + type.valueToDecimal(extreme); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + direct.valueToDecimal(extreme); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + BytesBuffer.allocate(16).writeProperty(DataType.DECIMAL, extreme); + }); + } + } + + @Test + public void testSerializerDoesNotExpandOutOfBoundsValues() { + // within the bounds: plain form, every digit + Assert.assertEquals("1000", BigDecimalSerializer.exactString(new BigDecimal("1E+3"))); + Assert.assertEquals("0." + new String(new char[127]).replace("\0", "0") + "1", + BigDecimalSerializer.exactString(new BigDecimal("1E-128"))); + // beyond them: the scientific form, exact but not expanded on the client + Assert.assertEquals("1E+999999999", + BigDecimalSerializer.exactString(new BigDecimal("1E+999999999"))); + Assert.assertEquals("1E-129", BigDecimalSerializer.exactString(new BigDecimal("1E-129"))); + Assert.assertEquals("{\"v\":1E+999999999}", + JsonUtil.toJson(ImmutableMap.of("v", new BigDecimal("1E+999999999")))); } @Test diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/ElementBuilder.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/ElementBuilder.java index e1d6c0818..190ce6d6d 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/ElementBuilder.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/ElementBuilder.java @@ -22,6 +22,8 @@ import java.nio.charset.CharsetEncoder; import java.nio.charset.CoderResult; import java.nio.charset.StandardCharsets; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -212,7 +214,42 @@ protected boolean retainField(String fieldName, Object fieldValue) { } return !nullableKeys.contains(mappedKey) || - !nullValues.contains(fieldValue); + !isNullValue(nullValues, fieldValue); + } + + /** + * Numbers are matched by value: JSON rows and the mapping's null_values + * are both parsed as BigDecimal now, whose equals() is scale-sensitive + * (1.0 vs 1.00); before, both were Double and matched. + */ + public static boolean isNullValue(Set nullValues, Object fieldValue) { + if (nullValues.contains(fieldValue)) { + return true; + } + if (!(fieldValue instanceof Number)) { + return false; + } + BigDecimal value = toBigDecimal((Number) fieldValue); + for (Object nullValue : nullValues) { + if (nullValue instanceof Number && + value.compareTo(toBigDecimal((Number) nullValue)) == 0) { + return true; + } + } + return false; + } + + private static BigDecimal toBigDecimal(Number number) { + if (number instanceof BigDecimal) { + return (BigDecimal) number; + } + if (number instanceof BigInteger) { + return new BigDecimal((BigInteger) number); + } + if (number instanceof Double || number instanceof Float) { + return new BigDecimal(number.toString()); + } + return BigDecimal.valueOf(number.longValue()); } protected void addProperty(GraphElement element, String key, Object value) { diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java index e8a1e1053..be504e8f1 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java @@ -363,6 +363,15 @@ private static boolean fromJsonParser(InputSource source) { return false; } + private static boolean withinDecimalBounds(BigDecimal decimal) { + try { + DataType.checkDecimalBounds(decimal); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + private static BigDecimal parseDecimal(String key, Object rawValue) { BigDecimal decimal; try { @@ -454,7 +463,9 @@ private static List split(String key, String rawValue, private static boolean checkDataType(String key, Object value, DataType dataType) { if (dataType.isDecimal()) { - return value instanceof BigDecimal; + // Only a BigDecimal within the DECIMAL bounds is already typed; + // anything else goes through parseDecimal, which rejects it + return value instanceof BigDecimal && withinDecimalBounds((BigDecimal) value); } if (value instanceof Number) { return parseNumber(key, value, dataType) != null; diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java index e31d6cd54..44f2881f1 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java @@ -99,6 +99,15 @@ public void testConvertDecimalList() { Assert.assertThrows(IllegalArgumentException.class, () -> { DataTypeUtil.convert(ImmutableList.of("x"), amounts, SOURCE); }); + // an already typed element outside the DECIMAL bounds is rejected + // like the same scalar + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataTypeUtil.convert(ImmutableList.of(new BigDecimal("1E+129")), amounts, SOURCE); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataTypeUtil.convert(ImmutableList.of(new BigDecimal("1E+129"), new BigDecimal("2")), + amounts, SOURCE); + }); } @Test diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java new file mode 100644 index 000000000..4a48d07ba --- /dev/null +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.loader.test.unit; + +import java.math.BigDecimal; +import java.util.Map; +import java.util.Set; + +import org.apache.hugegraph.loader.builder.ElementBuilder; +import org.apache.hugegraph.loader.mapping.ElementMapping; +import org.apache.hugegraph.loader.mapping.VertexMapping; +import org.apache.hugegraph.loader.util.JsonUtil; +import org.apache.hugegraph.testutil.Assert; +import org.junit.Test; + +import com.google.common.collect.ImmutableSet; + +public class NullValuesTest { + + @Test + public void testNumericNullValuesMatchByValue() { + // JSON rows and null_values are both parsed as BigDecimal now + ElementMapping mapping = JsonUtil.fromJson( + "{\"id\":\"name\",\"label\":\"person\",\"null_values\":[1.0,\"\"]}", VertexMapping.class); + Set nullValues = mapping.nullValues(); + Assert.assertTrue(nullValues.contains(new BigDecimal("1.0"))); + Map row = JsonUtil.fromJson("{\"a\":1.00,\"b\":1.5,\"c\":\"\"}", + Map.class); + Assert.assertTrue(ElementBuilder.isNullValue(nullValues, row.get("a"))); + Assert.assertFalse(ElementBuilder.isNullValue(nullValues, row.get("b"))); + Assert.assertTrue(ElementBuilder.isNullValue(nullValues, row.get("c"))); + // values from other parsers (JDBC, CSV) match by value too + Assert.assertTrue(ElementBuilder.isNullValue(nullValues, 1L)); + Assert.assertTrue(ElementBuilder.isNullValue(nullValues, 1.0d)); + Assert.assertFalse(ElementBuilder.isNullValue(nullValues, "1.0")); + Assert.assertFalse(ElementBuilder.isNullValue(ImmutableSet.of("NULL"), 1L)); + } +} diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java index ae66e5ccb..c674f6208 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java @@ -25,6 +25,7 @@ LineTest.class, DateUtilTest.class, DataTypeUtilTest.class, + NullValuesTest.class, MappingConverterTest.class, LoadProgressTest.class, RangesTimerTest.class From 991a97f7e188fd3acf267aa41788143e1d72bd61 Mon Sep 17 00:00:00 2001 From: Sebastian Gruza Date: Thu, 1 Oct 2026 08:51:22 +0000 Subject: [PATCH 6/8] fix(loader): keep JSON fraction mapping keys and signed zero, tolerate non-finite numbers and null decimal elements - JsonValueDeser: untyped JSON fractions stay BigDecimal except a negative zero, which BigDecimal cannot carry; it stays a Double so TEXT values and ids keep "-0.0". - DataTypeUtil.mappingKey: a JSON fraction looks up the mapping by the string the double path produced ("1.0" for 1.00), so existing mappings keep matching; any other BigDecimal keeps its own text. - isNullValue: a NaN or infinity from JDBC is not converted to BigDecimal (equal only to itself through contains()). - parseDecimal rejects a null element with a conversion error. Co-Authored-By: Claude Opus 4.8 --- .../loader/builder/ElementBuilder.java | 19 +++++-- .../hugegraph/loader/util/DataTypeUtil.java | 21 +++++++- .../hugegraph/loader/util/JsonUtil.java | 1 + .../hugegraph/loader/util/JsonValueDeser.java | 49 +++++++++++++++++++ .../loader/test/unit/DataTypeUtilTest.java | 17 +++++++ .../loader/test/unit/NullValuesTest.java | 26 ++++++++++ 6 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/JsonValueDeser.java diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/ElementBuilder.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/ElementBuilder.java index 190ce6d6d..d61506f6e 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/ElementBuilder.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/ElementBuilder.java @@ -230,15 +230,22 @@ public static boolean isNullValue(Set nullValues, Object fieldValue) { return false; } BigDecimal value = toBigDecimal((Number) fieldValue); + if (value == null) { + // NaN or an infinity: equal only to itself, handled by contains() + return false; + } for (Object nullValue : nullValues) { - if (nullValue instanceof Number && - value.compareTo(toBigDecimal((Number) nullValue)) == 0) { - return true; + if (nullValue instanceof Number) { + BigDecimal other = toBigDecimal((Number) nullValue); + if (other != null && value.compareTo(other) == 0) { + return true; + } } } return false; } + /** null for NaN and the infinities, which BigDecimal cannot hold. */ private static BigDecimal toBigDecimal(Number number) { if (number instanceof BigDecimal) { return (BigDecimal) number; @@ -247,6 +254,10 @@ private static BigDecimal toBigDecimal(Number number) { return new BigDecimal((BigInteger) number); } if (number instanceof Double || number instanceof Float) { + double d = number.doubleValue(); + if (Double.isNaN(d) || Double.isInfinite(d)) { + return null; + } return new BigDecimal(number.toString()); } return BigDecimal.valueOf(number.longValue()); @@ -306,7 +317,7 @@ protected Object mappingValue(String fieldName, Object fieldValue) { if (this.mapping().mappingValues().isEmpty()) { return fieldValue; } - String fieldStrValue = String.valueOf(fieldValue); + String fieldStrValue = DataTypeUtil.mappingKey(fieldValue, this.struct.input()); return this.mapping().mappingValue(fieldName, fieldStrValue, this.headerCaseSensitive()); } diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java index be504e8f1..90e3df372 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java @@ -211,7 +211,7 @@ private static Object parseSingleValue(String key, Object rawValue, // ("1.5", not "1.50"), so existing ids do not change. // A BigDecimal from any other source (JDBC DECIMAL/NUMERIC) // keeps its own text, as before - return Double.toString(((BigDecimal) value).doubleValue()); + return jsonNumberText((BigDecimal) value); } if (value instanceof Number) { return value.toString(); @@ -372,7 +372,26 @@ private static boolean withinDecimalBounds(BigDecimal decimal) { } } + /** + * The string a JSON fraction had before it was read as BigDecimal: the + * shortest double representation ("1.5" for 1.50, "1.0E-7" for 1e-7). + * TEXT values and mapping lookups keep the keys they always had. + */ + public static String jsonNumberText(BigDecimal value) { + return Double.toString(value.doubleValue()); + } + + /** The lookup key of a field value in the mapping's value map. */ + public static String mappingKey(Object fieldValue, InputSource source) { + if (fieldValue instanceof BigDecimal && fromJsonParser(source)) { + return jsonNumberText((BigDecimal) fieldValue); + } + return String.valueOf(fieldValue); + } + private static BigDecimal parseDecimal(String key, Object rawValue) { + E.checkArgument(rawValue != null, + "Failed to convert value(key='%s') null to Decimal", key); BigDecimal decimal; try { decimal = DataType.DECIMAL.valueToDecimal(rawValue); diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/JsonUtil.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/JsonUtil.java index ed4c4a587..4313dddd4 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/JsonUtil.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/JsonUtil.java @@ -61,6 +61,7 @@ public final class JsonUtil { SimpleModule module = new SimpleModule(); module.addDeserializer(InputSource.class, new InputSourceDeser()); module.addDeserializer(InputProgress.class, new InputProgressDeser()); + module.addDeserializer(Object.class, new JsonValueDeser()); registerModule(module); } diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/JsonValueDeser.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/JsonValueDeser.java new file mode 100644 index 000000000..93e56eb86 --- /dev/null +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/JsonValueDeser.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.loader.util; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer; + +/** + * Untyped JSON values (a line of a JSON file, a mapping) with fractions as + * BigDecimal, every digit kept for a DECIMAL column, except a negative zero: + * BigDecimal cannot carry its sign, so {@code -0.0} stays a {@code Double} + * and a TEXT column or an id built from it keeps "-0.0" as before. + */ +public class JsonValueDeser extends UntypedObjectDeserializer.Vanilla { + + private static final long serialVersionUID = 1L; + + @Override + public Object deserialize(JsonParser parser, DeserializationContext context) + throws IOException { + if (parser.currentToken() == JsonToken.VALUE_NUMBER_FLOAT) { + java.math.BigDecimal decimal = parser.getDecimalValue(); + if (decimal.signum() == 0 && parser.getText().startsWith("-")) { + return -0.0d; + } + return decimal; + } + return super.deserialize(parser, context); + } +} diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java index 44f2881f1..1fef9c976 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/DataTypeUtilTest.java @@ -99,6 +99,13 @@ public void testConvertDecimalList() { Assert.assertThrows(IllegalArgumentException.class, () -> { DataTypeUtil.convert(ImmutableList.of("x"), amounts, SOURCE); }); + // a null element is a conversion error, not a NullPointerException + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataTypeUtil.convert(java.util.Arrays.asList(new BigDecimal("1"), null), amounts, + SOURCE); + }, e -> { + Assert.assertContains("null to Decimal", e.getMessage()); + }); // an already typed element outside the DECIMAL bounds is rejected // like the same scalar Assert.assertThrows(IllegalArgumentException.class, () -> { @@ -150,6 +157,16 @@ public void testConvertDecimalFromJsonLine() { DataTypeUtil.convert(texts.get("c"), label, JSON_SOURCE)); Assert.assertEquals("7", DataTypeUtil.convert(texts.get("d"), label, JSON_SOURCE)); + // a negative zero keeps its sign as text (BigDecimal cannot carry + // it, so the parser hands it over as a Double) and is a decimal zero + Map zero = JsonUtil.convertMap("{\"z\": -0.0, \"p\": 0.0}", + String.class, Object.class); + Assert.assertEquals("-0.0", DataTypeUtil.convert(zero.get("z"), label, JSON_SOURCE)); + Assert.assertEquals("0.0", DataTypeUtil.convert(zero.get("p"), label, JSON_SOURCE)); + Assert.assertEquals(0, new BigDecimal("0").compareTo( + (BigDecimal) DataTypeUtil.convert(zero.get("z"), amount, JSON_SOURCE))); + Assert.assertEquals(new BigDecimal("0.0"), + DataTypeUtil.convert(zero.get("p"), amount, JSON_SOURCE)); } @Test diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java index 4a48d07ba..1db283bb3 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java @@ -24,6 +24,9 @@ import org.apache.hugegraph.loader.builder.ElementBuilder; import org.apache.hugegraph.loader.mapping.ElementMapping; import org.apache.hugegraph.loader.mapping.VertexMapping; +import org.apache.hugegraph.loader.source.file.FileFormat; +import org.apache.hugegraph.loader.source.file.FileSource; +import org.apache.hugegraph.loader.util.DataTypeUtil; import org.apache.hugegraph.loader.util.JsonUtil; import org.apache.hugegraph.testutil.Assert; import org.junit.Test; @@ -49,5 +52,28 @@ public void testNumericNullValuesMatchByValue() { Assert.assertTrue(ElementBuilder.isNullValue(nullValues, 1.0d)); Assert.assertFalse(ElementBuilder.isNullValue(nullValues, "1.0")); Assert.assertFalse(ElementBuilder.isNullValue(ImmutableSet.of("NULL"), 1L)); + // a JDBC NaN or infinity is not a null value unless listed itself + Assert.assertFalse(ElementBuilder.isNullValue(nullValues, Double.NaN)); + Assert.assertFalse(ElementBuilder.isNullValue(nullValues, Float.POSITIVE_INFINITY)); + Assert.assertTrue(ElementBuilder.isNullValue(ImmutableSet.of(Double.NaN), Double.NaN)); + Assert.assertFalse(ElementBuilder.isNullValue(ImmutableSet.of(Double.NaN), 1.0d)); + } + + /** A JSON fraction looks up the mapping by the string it always had. */ + @Test + public void testJsonFractionMappingKey() { + FileSource json = new FileSource(); + json.format(FileFormat.JSON); + FileSource csv = new FileSource(); + csv.format(FileFormat.CSV); + Map row = JsonUtil.fromJson( + "{\"a\":1.00,\"b\":1e-7,\"c\":-0.0,\"d\":7,\"e\":\"1.00\"}", Map.class); + Assert.assertEquals("1.0", DataTypeUtil.mappingKey(row.get("a"), json)); + Assert.assertEquals("1.0E-7", DataTypeUtil.mappingKey(row.get("b"), json)); + Assert.assertEquals("-0.0", DataTypeUtil.mappingKey(row.get("c"), json)); + Assert.assertEquals("7", DataTypeUtil.mappingKey(row.get("d"), json)); + Assert.assertEquals("1.00", DataTypeUtil.mappingKey(row.get("e"), json)); + // a BigDecimal from any other source keeps its own text + Assert.assertEquals("1.00", DataTypeUtil.mappingKey(new BigDecimal("1.00"), csv)); } } From 5255714cc4f476529175732018f4f2e1e76a209a Mon Sep 17 00:00:00 2001 From: Sebastian Gruza Date: Fri, 2 Oct 2026 10:42:01 +0000 Subject: [PATCH 7/8] fix(loader): a value_mapping lookup never replaces an unmapped value; string ids from JSON fractions keep the double text Co-Authored-By: Claude Opus 4.8 --- .../loader/builder/ElementBuilder.java | 9 ++--- .../loader/mapping/ElementMapping.java | 6 +++ .../hugegraph/loader/util/DataTypeUtil.java | 30 +++++++++++++- .../loader/test/unit/NullValuesTest.java | 39 +++++++++++++++++++ 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/ElementBuilder.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/ElementBuilder.java index d61506f6e..086116fad 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/ElementBuilder.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/ElementBuilder.java @@ -317,9 +317,8 @@ protected Object mappingValue(String fieldName, Object fieldValue) { if (this.mapping().mappingValues().isEmpty()) { return fieldValue; } - String fieldStrValue = DataTypeUtil.mappingKey(fieldValue, this.struct.input()); - return this.mapping().mappingValue(fieldName, fieldStrValue, - this.headerCaseSensitive()); + return DataTypeUtil.mapValue(this.mapping(), fieldName, fieldValue, this.struct.input(), + this.headerCaseSensitive()); } protected String mappingField(String fileName) { @@ -351,7 +350,7 @@ private void customizeId(VertexLabel vertexLabel, Vertex vertex, } if (idStrategy.isCustomizeString()) { - String id = (String) idValue.toString(); + String id = DataTypeUtil.idText(idValue, this.struct.input()); this.checkVertexIdLength(id); vertex.id(id); } else if (idStrategy.isCustomizeNumber()) { @@ -837,7 +836,7 @@ public void olapVertexId(VertexLabel vertexLabel, Vertex vertex, String idField, Object idValue) { IdStrategy idStrategy = vertexLabel.idStrategy(); if (idStrategy.isCustomizeString() || idStrategy.isPrimaryKey()) { - String id = (String) idValue.toString(); + String id = DataTypeUtil.idText(idValue, this.struct.input()); this.checkVertexIdLength(id); vertex.id(id); } else if (idStrategy.isCustomizeNumber() || idStrategy.isAutomatic()) { diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/mapping/ElementMapping.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/mapping/ElementMapping.java index 72f6ba529..f20c575bd 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/mapping/ElementMapping.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/mapping/ElementMapping.java @@ -169,6 +169,12 @@ public void mappingValues(Map> mappingValues) { this.mappingValues = mappingValues; } + /** The mapped value for a field, or null when the mapping has no entry for it. */ + public Object mappedValue(String fieldName, String rawValue, boolean caseSensitive) { + Object mapped = this.mappingValue(fieldName, rawValue, caseSensitive); + return mapped == (Object) rawValue ? null : mapped; + } + public Object mappingValue(String fieldName, String rawValue, boolean caseSensitive) { if (this.mappingValues.isEmpty()) { diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java index 90e3df372..ddb6744bd 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java @@ -31,6 +31,7 @@ import org.apache.hugegraph.util.ReflectionUtil; import org.apache.hugegraph.loader.constant.Constants; +import org.apache.hugegraph.loader.mapping.ElementMapping; import org.apache.hugegraph.loader.source.AbstractSource; import org.apache.hugegraph.loader.source.InputSource; import org.apache.hugegraph.loader.source.file.FileFormat; @@ -92,7 +93,7 @@ public static List splitField(String key, Object rawColumnValue, return (List) rawColumnValue; } // TODO: Seems a bit violent - String rawValue = rawColumnValue.toString(); + String rawValue = idText(rawColumnValue, source); return split(key, rawValue, source); } @@ -389,6 +390,33 @@ public static String mappingKey(Object fieldValue, InputSource source) { return String.valueOf(fieldValue); } + /** + * The mapped value when the mapping has an entry for this field value, + * else the original value: the lookup key is only a key, so a field + * without a mapping entry (e.g. a DECIMAL column of a struct that maps + * another field) keeps every digit. + */ + public static Object mapValue(ElementMapping mapping, String fieldName, Object fieldValue, + InputSource source, boolean caseSensitive) { + if (mapping.mappingValues().isEmpty()) { + return fieldValue; + } + Object mapped = mapping.mappedValue(fieldName, mappingKey(fieldValue, source), caseSensitive); + return mapped != null ? mapped : fieldValue; + } + + /** + * A field value as the text of an id: a JSON fraction gives the string + * the double path produced ("1.5" for 1.50, "100.0" for 1e2), so ids + * of an existing dataset do not change with the BigDecimal parsing. + */ + public static String idText(Object value, InputSource source) { + if (value instanceof BigDecimal && fromJsonParser(source)) { + return jsonNumberText((BigDecimal) value); + } + return value.toString(); + } + private static BigDecimal parseDecimal(String key, Object rawValue) { E.checkArgument(rawValue != null, "Failed to convert value(key='%s') null to Decimal", key); diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java index 1db283bb3..a642c9c8a 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java @@ -59,6 +59,45 @@ public void testNumericNullValuesMatchByValue() { Assert.assertFalse(ElementBuilder.isNullValue(ImmutableSet.of(Double.NaN), 1.0d)); } + /** A field without a mapping entry keeps its value; a DECIMAL column of a mapped struct keeps every digit. */ + @Test + public void testMappedStructKeepsUnmappedDecimal() { + FileSource json = new FileSource(); + json.format(FileFormat.JSON); + ElementMapping mapping = JsonUtil.fromJson( + "{\"id\":\"name\",\"label\":\"person\",\"value_mapping\":{\"city\":{\"1\":\"Beijing\"}}}", + VertexMapping.class); + Map row = JsonUtil.fromJson( + "{\"city\":1,\"amount\":12345678901234567890.10,\"ratio\":1.50}", Map.class); + Assert.assertEquals("Beijing", DataTypeUtil.mapValue(mapping, "city", row.get("city"), json, true)); + Object amount = DataTypeUtil.mapValue(mapping, "amount", row.get("amount"), json, true); + Assert.assertEquals(new BigDecimal("12345678901234567890.10"), amount); + Assert.assertEquals(new BigDecimal("1.50"), DataTypeUtil.mapValue(mapping, "ratio", row.get("ratio"), json, true)); + // a mapped fraction is looked up by its double text + ElementMapping m2 = JsonUtil.fromJson( + "{\"id\":\"name\",\"label\":\"person\",\"value_mapping\":{\"ratio\":{\"1.5\":\"half\"}}}", + VertexMapping.class); + Assert.assertEquals("half", DataTypeUtil.mapValue(m2, "ratio", row.get("ratio"), json, true)); + // without any mapping the value is untouched + ElementMapping m3 = JsonUtil.fromJson("{\"id\":\"name\",\"label\":\"person\"}", VertexMapping.class); + Assert.assertSame(row.get("amount"), DataTypeUtil.mapValue(m3, "amount", row.get("amount"), json, true)); + } + + /** A string id built from a JSON fraction keeps the text the double path produced. */ + @Test + public void testJsonFractionIdText() { + FileSource json = new FileSource(); + json.format(FileFormat.JSON); + FileSource csv = new FileSource(); + csv.format(FileFormat.CSV); + Map row = JsonUtil.fromJson("{\"a\":1.50,\"b\":1e2,\"c\":1e-7,\"d\":7}", Map.class); + Assert.assertEquals("1.5", DataTypeUtil.idText(row.get("a"), json)); + Assert.assertEquals("100.0", DataTypeUtil.idText(row.get("b"), json)); + Assert.assertEquals("1.0E-7", DataTypeUtil.idText(row.get("c"), json)); + Assert.assertEquals("7", DataTypeUtil.idText(row.get("d"), json)); + Assert.assertEquals("1.50", DataTypeUtil.idText(new BigDecimal("1.50"), csv)); + } + /** A JSON fraction looks up the mapping by the string it always had. */ @Test public void testJsonFractionMappingKey() { From 514c5b46e4ef544991fd46f87db5b8b126308bf7 Mon Sep 17 00:00:00 2001 From: Sebastian Gruza Date: Mon, 5 Oct 2026 10:13:34 +0000 Subject: [PATCH 8/8] fix(loader,client): mapped fractions keep the double text for TEXT and id outputs on non-JSON rows, unmapped values stay strings, plain DECIMAL form bounded by its expanded digits --- .../serializer/BigDecimalSerializer.java | 9 ++++- .../hugegraph/unit/DecimalDataTypeTest.java | 6 +++ .../hugegraph/loader/util/DataTypeUtil.java | 34 ++++++++++++++++- .../loader/test/unit/NullValuesTest.java | 38 +++++++++++++++++++ 4 files changed, 83 insertions(+), 4 deletions(-) diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java index 5db1f2430..9624d7839 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/BigDecimalSerializer.java @@ -56,8 +56,13 @@ public void serialize(BigDecimal value, JsonGenerator generator, */ public static String exactString(BigDecimal value) { int scale = value.scale(); - if (value.precision() <= DataType.DECIMAL_MAX_PRECISION && - scale >= -DataType.DECIMAL_MAX_SCALE && scale <= DataType.DECIMAL_MAX_SCALE) { + // the plain form of a negative scale carries the trailing zeros as + // digits (1E+128 has precision 1 but 129 plain digits), and the + // server reads an integer token with up to DECIMAL_MAX_PRECISION + // digits; a positive scale stays within the scale bound + int plainDigits = scale < 0 ? value.precision() - scale : value.precision(); + if (plainDigits <= DataType.DECIMAL_MAX_PRECISION && + scale <= DataType.DECIMAL_MAX_SCALE) { return value.toPlainString(); } return value.toString(); diff --git a/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java b/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java index 2874e36d6..91bb98602 100644 --- a/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java +++ b/hugegraph-client/src/test/java/org/apache/hugegraph/unit/DecimalDataTypeTest.java @@ -137,6 +137,12 @@ public void testSerializerDoesNotExpandOutOfBoundsValues() { Assert.assertEquals("1E+999999999", BigDecimalSerializer.exactString(new BigDecimal("1E+999999999"))); Assert.assertEquals("1E-129", BigDecimalSerializer.exactString(new BigDecimal("1E-129"))); + // the plain expansion of 1E+128 has 129 digits, above the server's precision cap + Assert.assertEquals("1E+128", BigDecimalSerializer.exactString(new BigDecimal("1E+128"))); + String plain127 = BigDecimalSerializer.exactString(new BigDecimal("1E+127")); + Assert.assertEquals(128, plain127.length()); + Assert.assertTrue(plain127.startsWith("1000")); + Assert.assertEquals("-1E+128", BigDecimalSerializer.exactString(new BigDecimal("-1E+128"))); Assert.assertEquals("{\"v\":1E+999999999}", JsonUtil.toJson(ImmutableMap.of("v", new BigDecimal("1E+999999999")))); } diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java index ddb6744bd..c335663cd 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java @@ -70,6 +70,10 @@ public static Object convert(Object value, PropertyKey propertyKey, String key = propertyKey.name(); DataType dataType = propertyKey.dataType(); + if (value instanceof MappedDecimal) { + // the struct's replacement: exact for DECIMAL, the double text otherwise + value = dataType.isDecimal() ? ((MappedDecimal) value).value : value.toString(); + } Cardinality cardinality = propertyKey.cardinality(); switch (cardinality) { case SINGLE: @@ -402,7 +406,33 @@ public static Object mapValue(ElementMapping mapping, String fieldName, Object f return fieldValue; } Object mapped = mapping.mappedValue(fieldName, mappingKey(fieldValue, source), caseSensitive); - return mapped != null ? mapped : fieldValue; + if (mapped != null) { + // a fractional replacement value comes from the struct's JSON as + // a BigDecimal: on a non-JSON row source it is marked so that a + // TEXT or id output gets the double text the struct gave before + // ("1.5"), while a DECIMAL output keeps every digit + return mapped instanceof BigDecimal && !fromJsonParser(source) + ? new MappedDecimal((BigDecimal) mapped) : mapped; + } + // unmapped: the string the loader passed on before (a JSON boolean or a + // JDBC timestamp next to a value_mapping still loads into a TEXT key); + // only a BigDecimal keeps its object, so a DECIMAL column keeps every digit + return fieldValue instanceof BigDecimal ? fieldValue : String.valueOf(fieldValue); + } + + /** A fractional value_mapping replacement on a non-JSON row source, see {@link #mapValue}. */ + public static final class MappedDecimal { + + public final BigDecimal value; + + public MappedDecimal(BigDecimal value) { + this.value = value; + } + + @Override + public String toString() { + return jsonNumberText(this.value); + } } /** @@ -414,7 +444,7 @@ public static String idText(Object value, InputSource source) { if (value instanceof BigDecimal && fromJsonParser(source)) { return jsonNumberText((BigDecimal) value); } - return value.toString(); + return value.toString(); // a MappedDecimal prints its double text } private static BigDecimal parseDecimal(String key, Object rawValue) { diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java index a642c9c8a..3df8f14e8 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/NullValuesTest.java @@ -28,6 +28,7 @@ import org.apache.hugegraph.loader.source.file.FileSource; import org.apache.hugegraph.loader.util.DataTypeUtil; import org.apache.hugegraph.loader.util.JsonUtil; +import org.apache.hugegraph.structure.schema.PropertyKey; import org.apache.hugegraph.testutil.Assert; import org.junit.Test; @@ -81,6 +82,43 @@ public void testMappedStructKeepsUnmappedDecimal() { // without any mapping the value is untouched ElementMapping m3 = JsonUtil.fromJson("{\"id\":\"name\",\"label\":\"person\"}", VertexMapping.class); Assert.assertSame(row.get("amount"), DataTypeUtil.mapValue(m3, "amount", row.get("amount"), json, true)); + // an unmapped non-decimal value next to a value_mapping is the string it was before, + // so a JSON boolean still loads into a TEXT key + Map row2 = JsonUtil.fromJson("{\"city\":1,\"active\":true}", Map.class); + Object active = DataTypeUtil.mapValue(mapping, "active", row2.get("active"), json, true); + Assert.assertEquals("true", active); + PropertyKey text = new PropertyKey.BuilderImpl("active", null).asText().build(); + Assert.assertEquals("true", DataTypeUtil.convert(active, text, json)); + } + + /** + * A fractional replacement value of a value_mapping comes from the struct's JSON as a + * BigDecimal; on a CSV row it still gives the double text for TEXT and id outputs + * ("1.5", as before) and every digit for a DECIMAL output. + */ + @Test + public void testMappedFractionOnACsvRow() { + FileSource csv = new FileSource(); + csv.format(FileFormat.CSV); + ElementMapping mapping = JsonUtil.fromJson( + "{\"id\":\"name\",\"label\":\"person\"," + + "\"value_mapping\":{\"ratio\":{\"1\":1.50,\"2\":12345678901234567890.10}}}", + VertexMapping.class); + Object mapped = DataTypeUtil.mapValue(mapping, "ratio", "1", csv, true); + Assert.assertEquals("1.5", DataTypeUtil.idText(mapped, csv)); + PropertyKey text = new PropertyKey.BuilderImpl("ratio", null).asText().build(); + Assert.assertEquals("1.5", DataTypeUtil.convert(mapped, text, csv)); + PropertyKey decimal = new PropertyKey.BuilderImpl("ratio", null).asDecimal().build(); + Assert.assertEquals(new BigDecimal("1.50"), DataTypeUtil.convert(mapped, decimal, csv)); + Object big = DataTypeUtil.mapValue(mapping, "ratio", "2", csv, true); + Assert.assertEquals(new BigDecimal("12345678901234567890.10"), DataTypeUtil.convert(big, decimal, csv)); + Assert.assertEquals("1.2345678901234567E19", DataTypeUtil.convert(big, text, csv)); + // on a JSON row the replacement is handled by the JSON rule as before + FileSource json = new FileSource(); + json.format(FileFormat.JSON); + Object mappedJson = DataTypeUtil.mapValue(mapping, "ratio", 1, json, true); + Assert.assertEquals(new BigDecimal("1.50"), mappedJson); + Assert.assertEquals("1.5", DataTypeUtil.convert(mappedJson, text, json)); } /** A string id built from a JSON fraction keeps the text the double path produced. */