Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.hugegraph.client;

import java.math.BigDecimal;
import java.util.Map;

import org.apache.hugegraph.exception.ServerException;
Expand All @@ -25,9 +26,11 @@
import org.apache.hugegraph.rest.RestClientConfig;
import org.apache.hugegraph.rest.RestHeaders;
import org.apache.hugegraph.rest.RestResult;
import org.apache.hugegraph.serializer.BigDecimalSerializer;
import org.apache.hugegraph.serializer.PathDeserializer;
import org.apache.hugegraph.structure.graph.Path;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.JsonUtilCommon;
import org.apache.hugegraph.util.VersionUtil;
import org.apache.hugegraph.util.VersionUtil.Version;

Expand All @@ -49,6 +52,11 @@ public class RestClient extends AbstractRestClient {
SimpleModule module = new SimpleModule();
module.addDeserializer(Path.class, new PathDeserializer());
RestResult.registerModule(module);

// Request bodies: a BigDecimal goes as a plain number, see JsonUtil
SimpleModule decimals = new SimpleModule();
decimals.addSerializer(BigDecimal.class, new BigDecimalSerializer());
JsonUtilCommon.registerModule(decimals);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ This registers the string serializer on the shared JsonUtilCommon mapper, so every BigDecimal in any request body now goes out as "1.5", not only values for DECIMAL keys.

On the server, a DOUBLE/FLOAT/LONG/INT key goes through DataType.valueToNumber, which returns null unless the value is a Number or one of the Infinity/NaN strings. So a client that today writes vertex.property("price", new BigDecimal("1.5")) to a asDouble() key (common when values come from JDBC NUMERIC columns) gets a 400 Invalid property value after this change, and a BigDecimal Gremlin binding turns into a string inside the script. The loader and spark connector are safe only because they convert to Double first.

Please either make the server accept numeric strings for numeric keys as part of #3209, or limit the string form to values headed for DECIMAL keys, and add an API test that writes a BigDecimal to a DOUBLE key. If the behaviour change is intended, it needs a line in the PR description and release notes under "public API".

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch, thanks: a client feeding JDBC NUMERIC into an asDouble() key would have got a 400. I changed the approach in a57545f: a BigDecimal is no longer a string but a plain JSON number in toPlainString() form (1000, not 1E+3; every digit). For numeric keys nothing changes against any server, since valueToNumber accepts any Number; for DECIMAL the exactness comes from the server side: in apache/hugegraph#3209 (0146849b) Jersey now reads JSON fractions as BigDecimal (ObjectMapperResolver, USE_BIG_DECIMAL_FOR_FLOATS), so a 39-digit literal reaches a DECIMAL key intact while a DOUBLE key narrows it to a double as before. VertexApiTest.testCreateWithBigDecimalOnDoubleKey writes new BigDecimal("1.5") to a DOUBLE key and reads 1.5 back; it runs in the regular ApiTestSuite, i.e. against the 1.7.0 server in CI as well. A Gremlin binding holding a BigDecimal stays a number. There is no public-behaviour change left to note in the release notes; the PR description's "public API" section is updated accordingly.

}

public RestClient(String url, String username, String password, int timeout) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/

package org.apache.hugegraph.serializer;

import java.io.IOException;
import java.math.BigDecimal;

import org.apache.hugegraph.structure.constant.DataType;

import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;

/**
* Write a BigDecimal as a plain JSON number ("1.10", never "1.1E+2").
* Jackson's default is the scientific form of BigDecimal.toString(); the
* plain form carries every digit, so a server that reads fractions as
* BigDecimal (apache/hugegraph#3209) stores a DECIMAL value exactly, and a
* server that reads them as double behaves as it always did. The value stays
* a number, so numeric keys (DOUBLE, FLOAT, LONG, INT) accept it too.
*/
public class BigDecimalSerializer extends StdSerializer<BigDecimal> {

private static final long serialVersionUID = 1L;

public BigDecimalSerializer() {
super(BigDecimal.class);
}

@Override
public void serialize(BigDecimal value, JsonGenerator generator,
SerializerProvider provider) throws IOException {
generator.writeNumber(exactString(value));
}

/**
* The plain form while the value is within the DECIMAL bounds the server
* accepts, else the scientific form: still exact and still a JSON number,
* but a value such as 1E+999999999 is not expanded into a billion
* characters on the client before the server rejects it.
*/
public static String exactString(BigDecimal value) {
int scale = value.scale();
if (value.precision() <= DataType.DECIMAL_MAX_PRECISION &&
scale >= -DataType.DECIMAL_MAX_SCALE && scale <= DataType.DECIMAL_MAX_SCALE) {
return value.toPlainString();

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Blocking: yes. Summary: 1E+128 passes this bound check (precision 1, scale -128), but toPlainString() emits a 129-digit integer; the server reads integer tokens as BigInteger and rejects precision 129. Evidence: the client DataType.valueToDecimal() accepts 1E+128, while apache/hugegraph#3209 PropertiesDeserializer uses getNumberValue() for VALUE_NUMBER_INT and server DataType.checkDecimalBounds() enforces the 128-digit cap. Please preserve exponent notation whenever plain expansion exceeds the server precision limit and add an upper-bound REST serialization test.

}
return value.toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.hugegraph.serializer.direct.struct;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Date;
import java.util.UUID;

Expand All @@ -42,18 +44,20 @@ public enum DataType {
TEXT(8, "text", String.class),
//BLOB(9, "blob", Blob.class),
DATE(10, "date", Date.class),
UUID(11, "uuid", UUID.class);
UUID(11, "uuid", UUID.class),
DECIMAL(12, "decimal", BigDecimal.class);

private final byte code;
private final String name;
private final Class<?> clazz;

// Must be initialized before the static block that fills it
static Table<Class<?>, Byte, DataType> TABLE = HashBasedTable.create();

static {
register(DataType.class);
}

static Table<Class<?>, Byte, DataType> TABLE = HashBasedTable.create();

static void register(Class<? extends DataType> clazz) {
Object enums;
try {
Expand Down Expand Up @@ -125,6 +129,10 @@ public boolean isUUID() {
return this == DataType.UUID;
}

public boolean isDecimal() {
return this == DataType.DECIMAL;
}

public <V> Number valueToNumber(V value) {
if (!(this.isNumber() && value instanceof Number)) {
return null;
Expand Down Expand Up @@ -192,6 +200,63 @@ public <V> UUID valueToUUID(V value) {
return null;
}

/*
* Bounds for a DECIMAL value, the same as the server applies: at most
* DECIMAL_MAX_PRECISION significant digits and an absolute scale of at
* most DECIMAL_MAX_SCALE. A value such as "1E+999999999" costs a few bytes
* to store and a billion characters on every read, and the direct loaders
* write bytes into storage without the server's check.
*/
public static final int DECIMAL_MAX_PRECISION = 128;
public static final int DECIMAL_MAX_SCALE = 128;

/**
* Convert a value to BigDecimal the same way the server does: BigDecimal
* as is, integral numbers exactly, any other Number and a decimal string
* through their decimal representation, then checked against the bounds.
*
* @return the BigDecimal, or null if the value can't be a decimal
*/
public <V> BigDecimal valueToDecimal(V value) {
if (!this.isDecimal()) {
return null;
}
BigDecimal decimal;
if (value instanceof BigDecimal) {
decimal = (BigDecimal) value;
} else if (value instanceof BigInteger) {
decimal = new BigDecimal((BigInteger) value);
} else if (value instanceof Byte || value instanceof Short ||
value instanceof Integer || value instanceof Long) {
decimal = BigDecimal.valueOf(((Number) value).longValue());
} else if (!(value instanceof Number) && !(value instanceof String)) {
return null;
} else {
try {
decimal = new BigDecimal(value.toString().trim());
} catch (NumberFormatException e) {
throw new IllegalArgumentException(String.format(
"Can't read '%s' as decimal", value));
}
}
return checkDecimalBounds(decimal);
}

public static BigDecimal checkDecimalBounds(BigDecimal decimal) {
// Compare the scale directly: Math.abs(Integer.MIN_VALUE) stays negative
int scale = decimal.scale();
int precision = decimal.precision();
if (precision > DECIMAL_MAX_PRECISION ||
scale < -DECIMAL_MAX_SCALE || scale > DECIMAL_MAX_SCALE) {
throw new IllegalArgumentException(String.format(
"Decimal value out of bounds: precision %d, scale %d " +
"(at most %d significant digits and a scale of at most " +
"%d in either direction)", precision, decimal.scale(),
DECIMAL_MAX_PRECISION, DECIMAL_MAX_SCALE));
}
return decimal;
}

public static DataType fromClass(Class<?> clazz) {
for (DataType type : DataType.values()) {
if (type.clazz() == clazz) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.math.BigDecimal;
import java.util.Date;
import java.util.UUID;

Expand Down Expand Up @@ -750,6 +751,13 @@ public void writeProperty(DataType dataType, Object value) {
this.writeLong(uuid.getMostSignificantBits());
this.writeLong(uuid.getLeastSignificantBits());
break;
case DECIMAL:
// Same layout as the server: unscaled two's-complement
// bytes followed by the scale, exact for any precision
BigDecimal decimal = dataType.valueToDecimal(value);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ The Javadoc on valueToDecimal says it converts "the same way the server does", but #3209 also runs DataType.checkDecimalBounds (at most 128 significant digits, scale at most 128 either way), and this copy does not. Through the REST path the server still rejects bad values, but the HBase direct loader writes these bytes straight into storage via HBaseSerializer, so a source value such as 1E+999999999 is stored in a few bytes, and every later server read calls toPlainString() on it and builds a string of about a billion characters. That is the case the server-side bound exists to stop.

Please apply the same bounds in both client valueToDecimal copies (or in BytesBuffer before writing), with constants matching #3209, and add a unit test that an out-of-bounds value is rejected.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done: both valueToDecimal copies end in checkDecimalBounds with the #3209 constants (DECIMAL_MAX_PRECISION = 128, DECIMAL_MAX_SCALE = 128), so BytesBuffer.writeProperty rejects 1E+999999999 before anything reaches HBase. DecimalDataTypeTest.testDecimalBounds covers 128 digits and 1E±128 as the boundary, 129 digits, 1E-129 and 1E+999999999 on both copies and on writeProperty.

this.writeBytes(decimal.unscaledValue().toByteArray());
this.writeVInt(decimal.scale());
break;
default:
//this.writeBytes(KryoUtil.toKryoWithType(value));
break;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
package org.apache.hugegraph.structure.constant;

import java.io.Serializable;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Date;
import java.util.UUID;

Expand All @@ -33,7 +35,14 @@ public enum DataType {
TEXT(8, "text", String.class),
BLOB(9, "blob", byte[].class),
DATE(10, "date", Date.class),
UUID(11, "uuid", UUID.class);
UUID(11, "uuid", UUID.class),
/*
* Arbitrary-precision decimal (java.math.BigDecimal), stored exactly by
* the server. On the wire: sent as a plain JSON number
* (BigDecimal.toPlainString(), every digit), returned by the server as a
* plain decimal string; new BigDecimal(String) restores it exactly
*/
DECIMAL(12, "decimal", BigDecimal.class);

private final byte code;
private final String name;
Expand Down Expand Up @@ -71,6 +80,68 @@ public boolean isUUID() {
return this == DataType.UUID;
}

public boolean isDecimal() {
return this == DataType.DECIMAL;
}

/*
* Bounds for a DECIMAL value, the same as the server applies: at most
* DECIMAL_MAX_PRECISION significant digits and an absolute scale of at
* most DECIMAL_MAX_SCALE. A value such as "1E+999999999" costs a few bytes
* to store and a billion characters on every read, and the direct loaders
* write bytes into storage without the server's check.
*/
public static final int DECIMAL_MAX_PRECISION = 128;
public static final int DECIMAL_MAX_SCALE = 128;

/**
* Convert a value to BigDecimal the same way the server does: BigDecimal
* as is, integral numbers exactly, any other Number and a decimal string
* through their decimal representation, then checked against the bounds.
*
* @return the BigDecimal, or null if the value can't be a decimal
* @throws IllegalArgumentException if the string is not a decimal number
*/
public <V> BigDecimal valueToDecimal(V value) {
if (!this.isDecimal()) {
return null;
}
BigDecimal decimal;
if (value instanceof BigDecimal) {
decimal = (BigDecimal) value;
} else if (value instanceof BigInteger) {
decimal = new BigDecimal((BigInteger) value);
} else if (value instanceof Byte || value instanceof Short ||
value instanceof Integer || value instanceof Long) {
decimal = BigDecimal.valueOf(((Number) value).longValue());
} else if (!(value instanceof Number) && !(value instanceof String)) {
return null;
} else {
try {
decimal = new BigDecimal(value.toString().trim());
} catch (NumberFormatException e) {
throw new IllegalArgumentException(String.format(
"Can't read '%s' as decimal", value));
}
}
return checkDecimalBounds(decimal);
}

public static BigDecimal checkDecimalBounds(BigDecimal decimal) {
// Compare the scale directly: Math.abs(Integer.MIN_VALUE) stays negative
int scale = decimal.scale();
int precision = decimal.precision();
if (precision > DECIMAL_MAX_PRECISION ||
scale < -DECIMAL_MAX_SCALE || scale > DECIMAL_MAX_SCALE) {
throw new IllegalArgumentException(String.format(
"Decimal value out of bounds: precision %d, scale %d " +
"(at most %d significant digits and a scale of at most " +
"%d in either direction)", precision, decimal.scale(),
DECIMAL_MAX_PRECISION, DECIMAL_MAX_SCALE));
}
return decimal;
}

public boolean isBoolean() {
return this == BOOLEAN;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ public interface Builder extends SchemaBuilder<PropertyKey> {

Builder asLong();

Builder asDecimal();

Builder cardinality(Cardinality cardinality);

Builder valueSingle();
Expand Down Expand Up @@ -266,6 +268,12 @@ public Builder asLong() {
return this;
}

@Override
public Builder asDecimal() {
this.propertyKey.dataType = DataType.DECIMAL;
return this;
}

@Override
public Builder cardinality(Cardinality cardinality) {
this.propertyKey.cardinality = cardinality;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,32 @@
package org.apache.hugegraph.util;

import java.io.IOException;
import java.math.BigDecimal;

import org.apache.hugegraph.rest.SerializeException;
import org.apache.hugegraph.serializer.BigDecimalSerializer;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.Module;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;

public final class JsonUtil {

private static final ObjectMapper MAPPER = new ObjectMapper();

static {
/*
* A BigDecimal travels as a plain JSON number ("1.10", not "1.1E+2")
* so that a DECIMAL key receives every digit. The same serializer is
* registered for request bodies in RestClient.
*/
SimpleModule module = new SimpleModule();
module.addSerializer(BigDecimal.class, new BigDecimalSerializer());
MAPPER.registerModule(module);
}

public static void registerModule(Module module) {
MAPPER.registerModule(module);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
VertexApiTest.class,
EdgeApiTest.class,
BatchUpdateElementApiTest.class,
DecimalPropertyApiTest.class,

GremlinApiTest.class,
VariablesApiTest.class,
Expand Down
Loading
Loading