diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/API.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/API.java index b50195b85d..c1ee404af8 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/API.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/API.java @@ -30,13 +30,16 @@ import org.apache.hugegraph.define.Checkable; import org.apache.hugegraph.exception.NotFoundException; import org.apache.hugegraph.metrics.MetricsUtil; +import org.apache.hugegraph.schema.PropertyKey; import org.apache.hugegraph.space.GraphSpace; import org.apache.hugegraph.space.SchemaTemplate; import org.apache.hugegraph.space.Service; +import org.apache.hugegraph.traversal.optimize.TraversalUtil; import org.apache.hugegraph.util.E; import org.apache.hugegraph.util.InsertionOrderUtil; import org.apache.hugegraph.util.JsonUtil; import org.apache.hugegraph.util.Log; +import org.apache.tinkerpop.gremlin.process.traversal.P; import org.slf4j.Logger; import com.codahale.metrics.Meter; @@ -222,6 +225,32 @@ protected static void checkUpdatingBody(Collection bodies) } @SuppressWarnings("unchecked") + /** + * Convert each filter value to the runtime type of its property key the + * way the traversal does (TraversalUtil.validPropertyValue, which knows + * the key's cardinality: a list on a single key converts every member, + * a scalar on a LIST/SET key stays a scalar for membership). A JSON + * fraction arrives as BigDecimal, so a DECIMAL value keeps every digit + * and a DOUBLE value becomes a double as before. Predicates (P.gt(...)) + * are converted when the traversal builds its conditions; values of + * unknown keys are left as they are. + */ + protected static void normalizeProperties(HugeGraph g, Map props) { + for (Map.Entry entry : props.entrySet()) { + Object value = entry.getValue(); + if (value == null || value instanceof P) { + continue; + } + PropertyKey pkey; + try { + pkey = g.propertyKey(entry.getKey()); + } catch (NotFoundException e) { + continue; + } + entry.setValue(TraversalUtil.validPropertyValue(value, pkey)); + } + } + protected static Map parseProperties(String properties) { if (properties == null || properties.isEmpty()) { return ImmutableMap.of(); @@ -229,7 +258,8 @@ protected static Map parseProperties(String properties) { Map props = null; try { - props = JsonUtil.fromJson(properties, Map.class); + // Exact fractions: a DECIMAL filter keeps every digit + props = JsonUtil.fromJsonExact(properties, Map.class); } catch (Exception ignored) { // ignore } diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/BatchAPI.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/BatchAPI.java index 85beb142db..a3b8a19bbe 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/BatchAPI.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/BatchAPI.java @@ -28,6 +28,7 @@ import org.apache.hugegraph.config.ServerOptions; import org.apache.hugegraph.define.Checkable; import org.apache.hugegraph.define.UpdateStrategy; +import org.apache.hugegraph.schema.PropertyKey; import org.apache.hugegraph.metrics.MetricsUtil; import org.apache.hugegraph.server.RestServer; import org.apache.hugegraph.structure.HugeElement; @@ -39,6 +40,7 @@ import com.codahale.metrics.Meter; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import io.swagger.v3.oas.annotations.media.Schema; @@ -92,6 +94,7 @@ protected abstract static class JsonElement implements Checkable { public String label; @Schema(description = "The properties of the vertex/edge in key-value format") @JsonProperty("properties") + @JsonDeserialize(using = PropertiesDeserializer.class) public Map properties; @Schema(description = "The type of element (vertex or edge)", hidden = true) @JsonProperty("type") @@ -108,6 +111,18 @@ protected abstract static class JsonElement implements Checkable { protected void updateExistElement(JsonElement oldElement, JsonElement newElement, Map strategies) { + this.updateExistElement(null, oldElement, newElement, strategies); + } + + /** + * Combine two JSON elements of the same id within one batch request. With + * a graph the raw JSON values are first normalised to the property key's + * data type (a decimal or a date arrives as a string), so the strategy + * sees typed values on both sides. + */ + protected void updateExistElement(HugeGraph g, JsonElement oldElement, + JsonElement newElement, + Map strategies) { if (oldElement == null) { return; } @@ -118,9 +133,15 @@ protected void updateExistElement(JsonElement oldElement, JsonElement newElement UpdateStrategy updateStrategy = kv.getValue(); if (oldElement.properties.get(key) != null && newElement.properties.get(key) != null) { - Object value = updateStrategy.checkAndUpdateProperty( - oldElement.properties.get(key), - newElement.properties.get(key)); + Object oldValue = oldElement.properties.get(key); + Object newValue = newElement.properties.get(key); + if (g != null) { + PropertyKey propertyKey = g.propertyKey(key); + oldValue = propertyKey.validValueOrThrow(oldValue); + newValue = propertyKey.validValueOrThrow(newValue); + } + Object value = updateStrategy.checkAndUpdateProperty(oldValue, + newValue); newElement.properties.put(key, value); } else if (oldElement.properties.get(key) != null && newElement.properties.get(key) == null) { @@ -142,10 +163,13 @@ protected void updateExistElement(HugeGraph g, Element oldElement, JsonElement n UpdateStrategy updateStrategy = kv.getValue(); if (oldElement.property(key).isPresent() && newElement.properties.get(key) != null) { + PropertyKey propertyKey = g.propertyKey(key); + // The stored value is typed; normalise the JSON one to match + Object newValue = propertyKey.validValueOrThrow( + newElement.properties.get(key)); Object value = updateStrategy.checkAndUpdateProperty( - oldElement.property(key).value(), - newElement.properties.get(key)); - value = g.propertyKey(key).validValueOrThrow(value); + oldElement.property(key).value(), newValue); + value = propertyKey.validValueOrThrow(value); newElement.properties.put(key, value); } else if (oldElement.property(key).isPresent() && newElement.properties.get(key) == null) { diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/EdgeAPI.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/EdgeAPI.java index 1f229cd6b1..6be2466a70 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/EdgeAPI.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/EdgeAPI.java @@ -201,7 +201,7 @@ public String update(@Context HugeConfig config, Id newEdgeId = getEdgeId(graph(manager, graphSpace, graph), newEdge); JsonEdge oldEdge = map.get(newEdgeId); - this.updateExistElement(oldEdge, newEdge, req.updateStrategies); + this.updateExistElement(g, oldEdge, newEdge, req.updateStrategies); map.put(newEdgeId, newEdge); }); @@ -341,6 +341,7 @@ public String list(@Context GraphManager manager, } } + normalizeProperties(g, props); for (Map.Entry entry : props.entrySet()) { traversal = traversal.has(entry.getKey(), entry.getValue()); } diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/PropertiesDeserializer.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/PropertiesDeserializer.java new file mode 100644 index 0000000000..0a188caace --- /dev/null +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/PropertiesDeserializer.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.api.graph; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonMappingException; + +/** + * Reads the "properties" object of a vertex or edge body so that a JSON + * fraction keeps every digit: it becomes a BigDecimal instead of a double, + * which is what a DECIMAL property key needs and what every numeric key + * narrows through DataType.valueToNumber as before. Only property values + * are read this way; the rest of the request body (job parameters, schema + * userdata, query options) keeps Jackson's default number types. + */ +public class PropertiesDeserializer extends JsonDeserializer> { + + @Override + public Map deserialize(JsonParser parser, + DeserializationContext ctxt) + throws IOException { + JsonToken token = parser.currentToken(); + if (token == JsonToken.VALUE_NULL) { + return null; + } + if (token != JsonToken.START_OBJECT) { + throw JsonMappingException.from(parser, + "Expected an object for 'properties', but got " + token); + } + return readObject(parser); + } + + private static Map readObject(JsonParser parser) + throws IOException { + Map object = new LinkedHashMap<>(); + while (parser.nextToken() != JsonToken.END_OBJECT) { + String name = parser.currentName(); + parser.nextToken(); + object.put(name, readValue(parser)); + } + return object; + } + + private static List readArray(JsonParser parser) + throws IOException { + List array = new ArrayList<>(); + while (parser.nextToken() != JsonToken.END_ARRAY) { + array.add(readValue(parser)); + } + return array; + } + + /** One value at the parser's current token: exact fractions, nested objects and arrays. */ + public static Object readValue(JsonParser parser) throws IOException { + JsonToken token = parser.currentToken(); + switch (token) { + case START_OBJECT: + return readObject(parser); + case START_ARRAY: + return readArray(parser); + case VALUE_STRING: + return parser.getText(); + case VALUE_NUMBER_INT: + return parser.getNumberValue(); + case VALUE_NUMBER_FLOAT: + // Exact: the literal's digits, not the nearest double + return parser.getDecimalValue(); + case VALUE_TRUE: + return Boolean.TRUE; + case VALUE_FALSE: + return Boolean.FALSE; + case VALUE_NULL: + return null; + default: + throw JsonMappingException.from(parser, + "Unexpected token in 'properties': " + token); + } + } +} diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/VertexAPI.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/VertexAPI.java index af1433ac46..e035f1f1b7 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/VertexAPI.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/graph/VertexAPI.java @@ -167,7 +167,7 @@ public String update(@Context HugeConfig config, req.jsonVertices.forEach(newVertex -> { Id newVertexId = getVertexId(g, newVertex); JsonVertex oldVertex = map.get(newVertexId); - this.updateExistElement(oldVertex, newVertex, req.updateStrategies); + this.updateExistElement(g, oldVertex, newVertex, req.updateStrategies); map.put(newVertexId, newVertex); }); @@ -286,6 +286,7 @@ public String list(@Context GraphManager manager, } } + normalizeProperties(g, props); for (Map.Entry entry : props.entrySet()) { traversal = traversal.has(entry.getKey(), entry.getValue()); } diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/schema/PropertyKeyAPI.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/schema/PropertyKeyAPI.java index 27d6ab1da2..64508a5e84 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/schema/PropertyKeyAPI.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/schema/PropertyKeyAPI.java @@ -46,6 +46,7 @@ import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.google.common.collect.ImmutableMap; import io.swagger.v3.oas.annotations.Parameter; @@ -254,6 +255,7 @@ private static class JsonPropertyKey implements Checkable { public String[] properties; @Schema(description = "User-defined metadata") @JsonProperty("user_data") + @JsonDeserialize(using = UserdataDeserializer.class) public Userdata userdata; @Schema(description = "Whether to check if property key exists before creation") @JsonProperty("check_exist") diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/schema/UserdataDeserializer.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/schema/UserdataDeserializer.java new file mode 100644 index 0000000000..887c4a374f --- /dev/null +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/schema/UserdataDeserializer.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.api.schema; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.hugegraph.api.graph.PropertiesDeserializer; +import org.apache.hugegraph.schema.Userdata; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +/** + * The userdata of a property key: {@code ~default_value} is read like an + * element property (a JSON fraction becomes a BigDecimal with every digit, + * so a DECIMAL default reaches the key exactly), every other entry keeps + * Jackson's default types, so custom metadata such as {@code {"rate":0.85}} + * stays a double and round-trips as a JSON number. + */ +public class UserdataDeserializer extends StdDeserializer { + + private static final long serialVersionUID = 1L; + + public UserdataDeserializer() { + super(Userdata.class); + } + + @Override + public Userdata deserialize(JsonParser parser, DeserializationContext context) + throws IOException { + JsonToken token = parser.currentToken(); + if (token == JsonToken.VALUE_NULL) { + return null; + } + if (token != JsonToken.START_OBJECT) { + throw JsonMappingException.from(parser, + "Expected an object for 'user_data', but got " + token); + } + Map map = new LinkedHashMap<>(); + while (parser.nextToken() != JsonToken.END_OBJECT) { + String name = parser.currentName(); + parser.nextToken(); + if (Userdata.DEFAULT_VALUE.equals(name)) { + map.put(name, PropertiesDeserializer.readValue(parser)); + } else { + map.put(name, context.readValue(parser, Object.class)); + } + } + return new Userdata(map); + } +} diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java index 7ed4efcd66..f4080f7bea 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java @@ -356,7 +356,8 @@ private enum ValueType { FLOAT(DataType.FLOAT), DOUBLE(DataType.DOUBLE), DATE(DataType.DATE), - UUID(DataType.UUID); + UUID(DataType.UUID), + DECIMAL(DataType.DECIMAL); private final DataType dataType; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/Condition.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/Condition.java index e77f2f0b1e..ca4b7196ed 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/Condition.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/Condition.java @@ -18,6 +18,7 @@ package org.apache.hugegraph.backend.query; import java.util.ArrayList; +import java.math.BigDecimal; import java.util.Collection; import java.util.Date; import java.util.List; @@ -77,12 +78,12 @@ public enum RelationType implements BiPredicate { IN("in", null, Collection.class, (v1, v2) -> { assert v2 != null; - return ((Collection) v2).contains(v1); + return collectionContains((Collection) v2, v1); }), NOT_IN("notin", null, Collection.class, (v1, v2) -> { assert v2 != null; - return !((Collection) v2).contains(v1); + return !collectionContains((Collection) v2, v1); }), PREFIX("prefix", Id.class, Id.class, (v1, v2) -> { @@ -280,6 +281,29 @@ public boolean test(Object first, Object second) { return this.tester.apply(first, second); } + /** + * Membership by value once a BigDecimal is involved: its equals() is + * scale sensitive (1.0 vs 1.00), while EQ compares decimals by value, + * so IN/NOT_IN do the same. Other numbers keep contains() semantics + * (an Integer 1 is not a Double 1.0 here, as before). + */ + private static boolean collectionContains(Collection values, Object value) { + if (values.contains(value)) { + return true; + } + if (!(value instanceof Number)) { + return false; + } + boolean decimal = value instanceof BigDecimal; + for (Object member : values) { + if (member instanceof Number && (decimal || member instanceof BigDecimal) && + NumericUtil.compareNumber(value, (Number) member) == 0) { + return true; + } + } + return false; + } + public boolean isRangeType() { return RANGE_TYPES.contains(this); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java index 097e98df19..6393c5f826 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java @@ -947,6 +947,11 @@ private static boolean numberEquals(Object number1, Object number2) { // Otherwise convert to BigDecimal to make two numbers comparable Number n1 = NumericUtil.convertToNumber(number1); Number n2 = NumericUtil.convertToNumber(number2); + if (n1 instanceof BigDecimal || n2 instanceof BigDecimal) { + // Exact: a decimal must not be squeezed through a double + return new BigDecimal(n1.toString()) + .compareTo(new BigDecimal(n2.toString())) == 0; + } BigDecimal b1 = BigDecimal.valueOf(n1.doubleValue()); BigDecimal b2 = BigDecimal.valueOf(n2.doubleValue()); return b1.compareTo(b2) == 0; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/serializer/QueryAdapter.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/serializer/QueryAdapter.java index 47b7c76a00..de10ce1ba4 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/serializer/QueryAdapter.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/serializer/QueryAdapter.java @@ -18,6 +18,8 @@ package org.apache.hugegraph.backend.query.serializer; import java.lang.reflect.Type; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -48,8 +50,13 @@ public class QueryAdapter extends AbstractSerializerAdapter { .build(); static boolean isPrimitive(Class clz) { + // Values whose class must travel with them: Gson reads an untyped + // JSON number back as a double, which would round a BigDecimal + if (clz == Date.class || clz == BigDecimal.class || clz == BigInteger.class) { + return true; + } try { - return (clz == Date.class) || ((Class) clz.getField("TYPE").get(null)).isPrimitive(); + return ((Class) clz.getField("TYPE").get(null)).isPrimitive(); } catch (Exception e) { return false; } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java index 0bb07760a5..92038e47ba 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java @@ -1237,8 +1237,8 @@ private void readUserdata(SchemaElement schema) { byte[] userdataBytes = column(HugeKeys.USER_DATA); String userdataStr = StringEncoding.decode(userdataBytes); @SuppressWarnings("unchecked") - Map userdata = JsonUtil.fromJson(userdataStr, - Map.class); + Map userdata = JsonUtil.fromJsonExact(userdataStr, + Map.class); for (Map.Entry e : userdata.entrySet()) { schema.userdata(e.getKey(), e.getValue()); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java index faf1508299..2b8ccf3ffc 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java @@ -17,6 +17,8 @@ package org.apache.hugegraph.backend.serializer; +import java.math.BigDecimal; +import java.math.BigInteger; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Arrays; @@ -660,6 +662,13 @@ public void writeProperty(DataType dataType, Object value) { this.writeLong(uuid.getMostSignificantBits()); this.writeLong(uuid.getLeastSignificantBits()); break; + case DECIMAL: + // unscaled two's-complement bytes + scale: exact for any + // precision, 33 bytes for a 78-digit (uint256) value + BigDecimal decimal = (BigDecimal) value; + this.writeBytes(decimal.unscaledValue().toByteArray()); + this.writeVInt(decimal.scale()); + break; default: // TODO: replace Kryo with Fury (https://github.com/apache/fury) this.writeBytes(KryoUtil.toKryoWithType(value)); @@ -693,6 +702,9 @@ public Object readProperty(DataType dataType) { return Blob.wrap(this.readBigBytes()); case UUID: return new UUID(this.readLong(), this.readLong()); + case DECIMAL: + BigInteger unscaled = new BigInteger(this.readBytes()); + return new BigDecimal(unscaled, this.readVInt()); default: // TODO: replace Kryo with Fury (https://github.com/apache/fury) return KryoUtil.fromKryoWithType(this.readBytes()); @@ -872,7 +884,7 @@ public BinaryId parseOlapId(HugeType type, boolean isOlap) { } // Parse id from bytes int start = this.buffer.position(); - // OLAP {PropertyKey}{VertexId} + // OLAP {PropertyKey}{VertexId} if (isOlap) { // Read olap property id first Id pkId = this.readId(); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java index 2d5cb81ec1..9276090fb9 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java @@ -911,7 +911,7 @@ private static void readUserdata(SchemaElement schema, // Parse all user data of a schema element String userdataStr = entry.column(HugeKeys.USER_DATA); @SuppressWarnings("unchecked") - Map userdata = JsonUtil.fromJson(userdataStr, + Map userdata = JsonUtil.fromJsonExact(userdataStr, Map.class); for (Map.Entry e : userdata.entrySet()) { schema.userdata(e.getKey(), e.getValue()); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/io/HugeGraphSONModule.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/io/HugeGraphSONModule.java index ddb7c1a981..a2cc8b387d 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/io/HugeGraphSONModule.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/io/HugeGraphSONModule.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.IOException; +import java.math.BigDecimal; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -51,6 +52,7 @@ import org.apache.hugegraph.structure.HugeElement; import org.apache.hugegraph.structure.HugeProperty; import org.apache.hugegraph.structure.HugeVertex; +import org.apache.hugegraph.type.define.DataType; import org.apache.hugegraph.type.define.HugeKeys; import org.apache.hugegraph.util.Blob; import org.apache.hugegraph.util.Log; @@ -183,6 +185,10 @@ public static void registerCommonSerializers(SimpleModule module) { module.addSerializer(Blob.class, new BlobSerializer()); module.addDeserializer(Blob.class, new BlobDeserializer()); + + // Decimals travel as strings: JSON numbers are doubles to most clients + module.addSerializer(BigDecimal.class, new BigDecimalSerializer()); + module.addDeserializer(BigDecimal.class, new BigDecimalDeserializer()); } public static void registerIdSerializers(SimpleModule module) { @@ -956,4 +962,72 @@ public Blob deserialize(JsonParser jsonParser, return Blob.wrap(bytes); } } + + private static class BigDecimalSerializer extends StdSerializer { + + public BigDecimalSerializer() { + super(BigDecimal.class); + } + + @Override + public void serialize(BigDecimal decimal, JsonGenerator jsonGenerator, + SerializerProvider provider) throws IOException { + jsonGenerator.writeString(exactString(decimal)); + } + + /** + * The plain form ("1000", "0.000000000000000001") while the scale + * is within the DECIMAL bound, so a stored value always reads as + * digits; beyond it the scientific form ("1E+999999999"), which is + * just as exact but does not expand the exponent into characters. + * A generic Gremlin result is not bound by the property check. + */ + static String exactString(BigDecimal decimal) { + int scale = decimal.scale(); + if (scale >= -DataType.DECIMAL_MAX_SCALE && + scale <= DataType.DECIMAL_MAX_SCALE) { + return decimal.toPlainString(); + } + return decimal.toString(); + } + + @Override + public void serializeWithType(BigDecimal decimal, + JsonGenerator jsonGenerator, + SerializerProvider provider, + TypeSerializer typeSer) + throws IOException { + /* + * The typed GraphSON mappers (v2/v3) call this variant and + * StdSerializer does not implement it. Keep the type prefix so + * that the value stays "gx:BigDecimal", but carry the plain + * string inside it: a JSON number would be read as a double by + * most clients, which is what this type exists to avoid. + */ + WritableTypeId typeId = typeSer.typeId(decimal, + JsonToken.VALUE_STRING); + typeSer.writeTypePrefix(jsonGenerator, typeId); + this.serialize(decimal, jsonGenerator, provider); + typeSer.writeTypeSuffix(jsonGenerator, typeId); + } + } + + private static class BigDecimalDeserializer extends StdDeserializer { + + public BigDecimalDeserializer() { + super(BigDecimal.class); + } + + @Override + public BigDecimal deserialize(JsonParser jsonParser, + DeserializationContext ctxt) + throws IOException { + JsonToken token = jsonParser.getCurrentToken(); + if (token == JsonToken.VALUE_NUMBER_INT || + token == JsonToken.VALUE_NUMBER_FLOAT) { + return jsonParser.getDecimalValue(); + } + return new BigDecimal(jsonParser.getText().trim()); + } + } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/PropertyKey.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/PropertyKey.java index 5bf34ea530..cf7b92f2b5 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/PropertyKey.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/PropertyKey.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.schema; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -116,7 +117,70 @@ public PropertyKey properties(Id... properties) { public void defineDefaultValue(Object value) { // TODO add a field default_value - this.userdata().put(Userdata.DEFAULT_VALUE, value); + this.userdata().put(Userdata.DEFAULT_VALUE, this.normalizeDefaultValue(value)); + } + + /** + * A DECIMAL key keeps its default value in userdata as the exact + * BigDecimal (a JSON fraction arrives as BigDecimal, a string is parsed), + * both when the key is defined through the API and when it is read back + * from the backend, so the value serializes the same way on every path. + * Every other data type keeps the raw value the user sent, as before: + * {@link #defaultValue()} converts it lazily when it is applied; only a + * BigDecimal (which exists solely because the API reads the default + * exactly) becomes the Double the default parser produced on master. + */ + /** A BigDecimal from the exact parser as the Double the default parser gave. */ + private static Object undoExact(Object value) { + if (value instanceof BigDecimal) { + return ((BigDecimal) value).doubleValue(); + } + if (value instanceof Collection) { + List values = new ArrayList<>(((Collection) value).size()); + for (Object member : (Collection) value) { + values.add(undoExact(member)); + } + return value instanceof Set ? new LinkedHashSet<>(values) : values; + } + return value; + } + + @Override + public void userdata(String key, Object value) { + if (Userdata.DEFAULT_VALUE.equals(key)) { + value = this.normalizeDefaultValue(value); + } + super.userdata(key, value); + } + + @Override + public void userdata(Userdata userdata) { + E.checkArgumentNotNull(userdata, "userdata"); + for (Map.Entry e : userdata.entrySet()) { + this.userdata(e.getKey(), e.getValue()); + } + } + + private Object normalizeDefaultValue(Object value) { + if (value == null) { + return value; + } + if (this.dataType != DataType.DECIMAL) { + return undoExact(value); + } + Object raw = value; + if (this.cardinality == Cardinality.SET && value instanceof Collection && + !(value instanceof Set)) { + raw = new LinkedHashSet<>((Collection) value); + } + try { + Object valid = this.validValue(raw); + return valid != null ? valid : value; + } catch (RuntimeException e) { + // An already stored default that does not convert keeps loading; + // the error surfaces when the default is applied, as before + return value; + } } public Object defaultValue() { @@ -311,8 +375,11 @@ private V convValue(V value) { if (value == null) { return null; } - if (this.checkValueType(value)) { - // Same as expected type, no conversion required + if (this.checkValueType(value) && !this.dataType().isDecimal()) { + // Same as expected type, no conversion required. A decimal is + // not short-circuited: a ready-made BigDecimal (Gremlin literal, + // SUM result of a batch update) still has to pass the bounds + // check in DataType.valueToDecimal() return value; } @@ -368,6 +435,10 @@ private V convSingleValue(V value) { @SuppressWarnings("unchecked") V blob = (V) this.dataType().valueToBlob(value); return blob; + } else if (this.dataType().isDecimal()) { + @SuppressWarnings("unchecked") + V decimal = (V) this.dataType().valueToDecimal(value); + return decimal; } if (this.checkDataType(value)) { @@ -400,6 +471,8 @@ public interface Builder extends SchemaBuilder { Builder asLong(); + Builder asDecimal(); + Builder valueSingle(); Builder valueList(); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/EdgeLabelBuilder.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/EdgeLabelBuilder.java index 32937a2cf0..66ae115ad4 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/EdgeLabelBuilder.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/EdgeLabelBuilder.java @@ -602,6 +602,10 @@ private void checkSortKeys() { "The sort key '%s' must be contained in " + "properties '%s' for edge label '%s'", key, this.name, this.properties); + PropertyKey propertyKey = this.graph().propertyKey(key); + E.checkArgument(!propertyKey.dataType().isDecimal(), + "The sort key '%s' of edge label '%s' can't " + + "be a decimal property", key, this.name); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/IndexLabelBuilder.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/IndexLabelBuilder.java index 397df66229..7492d30fe1 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/IndexLabelBuilder.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/IndexLabelBuilder.java @@ -114,6 +114,12 @@ public IndexLabel build() { indexLabel.indexType(this.indexType); for (String field : this.indexFields) { PropertyKey propertyKey = graph.propertyKey(field); + // Also guarded in checkFields(), but build() is reached directly + // by the OLAP property-key path, which skips checkFields() + E.checkArgument(!propertyKey.dataType().isDecimal(), + "Not allowed to build index on property key " + + "'%s' whose data type is decimal", + propertyKey.name()); indexLabel.indexField(propertyKey.id()); } indexLabel.userdata(this.userdata); @@ -472,6 +478,9 @@ private void checkFields(Set propertyIds) { E.checkArgument(pkey.aggregateType().isIndexable(), "The aggregate type %s is not indexable", pkey.aggregateType()); + E.checkArgument(!pkey.dataType().isDecimal(), + "Not allowed to build index on property key " + + "'%s' whose data type is decimal", pkey.name()); if (pkey.cardinality().multiple()) { E.checkArgument(fields.size() == 1, diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/PropertyKeyBuilder.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/PropertyKeyBuilder.java index a50b426d1d..1a418342f0 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/PropertyKeyBuilder.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/PropertyKeyBuilder.java @@ -273,6 +273,12 @@ public PropertyKeyBuilder asDouble() { return this; } + @Override + public PropertyKeyBuilder asDecimal() { + this.dataType = DataType.DECIMAL; + return this; + } + @Override public PropertyKeyBuilder asFloat() { this.dataType = DataType.FLOAT; @@ -427,7 +433,8 @@ private void checkAggregateType() { } if (this.aggregateType.isNumber() && - !this.dataType.isNumber() && !this.dataType.isDate()) { + !this.dataType.isNumber() && !this.dataType.isDecimal() && + !this.dataType.isDate()) { throw new NotAllowException( "Not allowed to set aggregate type '%s' for " + "property key '%s' with data type '%s'", @@ -452,6 +459,16 @@ private void checkOlap() { "property key '%s'", this.aggregateType, this.name); } + if (this.dataType.isDecimal() && + this.writeType != WriteType.OLAP_COMMON) { + // OLAP_SECONDARY / OLAP_RANGE build an index label on the key, + // and no index of any type is allowed on a decimal + throw new NotAllowException( + "Not allowed to set write type to %s for property key " + + "'%s' with data type '%s': decimal keys can't be indexed", + this.writeType, this.name, this.dataType); + } + if (this.writeType == WriteType.OLAP_RANGE && !this.dataType.isNumber() && !this.dataType.isDate()) { throw new NotAllowException( diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/VertexLabelBuilder.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/VertexLabelBuilder.java index 4962646209..5b3e25f55b 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/VertexLabelBuilder.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/VertexLabelBuilder.java @@ -512,6 +512,13 @@ private void checkPrimaryKeys() { "The primary key '%s' of vertex label '%s' " + "must be contained in properties: %s", key, this.name, this.properties); + // A primary key becomes part of the vertex id through + // LongEncoding/NumericUtil, which is lossy for a decimal + // (fractions collapse into a double, uint256 overflows a long) + PropertyKey propertyKey = this.graph().propertyKey(key); + E.checkArgument(!propertyKey.dataType().isDecimal(), + "The primary key '%s' of vertex label '%s' " + + "can't be a decimal property", key, this.name); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtil.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtil.java index e6a56027a1..861a59e743 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtil.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtil.java @@ -1254,7 +1254,7 @@ public static Query.Order convOrder(Order order) { return order == Order.desc ? Query.Order.DESC : Query.Order.ASC; } - private static V validPropertyValue(V value, PropertyKey pkey) { + public static V validPropertyValue(V value, PropertyKey pkey) { if (pkey.cardinality().single() && value instanceof Collection && !pkey.dataType().isBlob()) { // Expect single but got collection, like P.within([]) @@ -1488,7 +1488,9 @@ public static Condition parsePredicate(PropertyKey pk, String predicate) { private static Number predicateNumber(String value) { try { - return JsonUtil.fromJson(value, Number.class); + // Exact: a fraction becomes a BigDecimal with every digit and + // is converted to the property key's type by validPropertyValue + return JsonUtil.fromJsonExact(value, Number.class); } catch (Exception e) { // Try to parse date if (e.getMessage().contains("not a valid number") || @@ -1537,7 +1539,7 @@ private static Number[] predicateNumbers(String value, int count) { @SuppressWarnings("unchecked") private static V predicateArg(String value) { try { - return (V) JsonUtil.fromJson(value, Object.class); + return (V) JsonUtil.fromJsonExact(value, Object.class); } catch (Exception e) { throw new HugeException( "Invalid value '%s', expect a single value", e, value); @@ -1547,7 +1549,7 @@ private static V predicateArg(String value) { @SuppressWarnings("unchecked") private static List predicateArgs(String value) { try { - return JsonUtil.fromJson("[" + value + "]", List.class); + return JsonUtil.fromJsonExact("[" + value + "]", List.class); } catch (Exception e) { throw new HugeException( "Invalid value '%s', expect a list", e, value); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/type/define/DataType.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/type/define/DataType.java index 2bfa93e7d7..9a1bf987c0 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/type/define/DataType.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/type/define/DataType.java @@ -17,6 +17,8 @@ package org.apache.hugegraph.type.define; +import java.math.BigDecimal; +import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.Date; import java.util.List; @@ -43,7 +45,14 @@ public enum DataType implements SerialEnum { TEXT(8, "text", String.class), BLOB(9, "blob", Blob.class), DATE(10, "date", Date.class), - UUID(11, "uuid", UUID.class); + UUID(11, "uuid", UUID.class), + /* + * Arbitrary-precision decimal (java.math.BigDecimal). Stored exactly; not a + * "number" in the isNumber() sense because it has no fixed-width, sortable + * encoding, so it can't be a sort key, a range/secondary index field or an + * OLAP range property. + */ + DECIMAL(12, "decimal", BigDecimal.class); private final byte code; private final String name; @@ -103,6 +112,10 @@ public boolean isUUID() { return this == DataType.UUID; } + public boolean isDecimal() { + return this == DataType.DECIMAL; + } + public Number valueToNumber(V value) { if (!(this.isNumber() && value instanceof Number) && !JsonUtil.isInfinityOrNaN(value)) { @@ -143,6 +156,67 @@ public Number valueToNumber(V value) { return number; } + /* + * Bounds for a DECIMAL value: at most DECIMAL_MAX_PRECISION significant + * digits and an absolute scale of at most DECIMAL_MAX_SCALE. uint256 + * with 18 fraction digits is 96 digits, so both fit with room to spare, + * while "1E+999999999" (a few bytes on disk, a billion characters from + * toPlainString() on every read) is rejected before it is stored. + */ + public static final int DECIMAL_MAX_PRECISION = 128; + public static final int DECIMAL_MAX_SCALE = 128; + + /** + * Convert a value to BigDecimal: BigDecimal as is, any other Number and a + * decimal string through their exact decimal representation. Float and + * Double go through Number.toString(), i.e. the shortest string that + * round-trips the binary value, so a client that already holds a lossy + * double gets that double, exactly. + * + * @return the BigDecimal, or null if the value is not a Number or String + * @throws IllegalArgumentException if the string is not a decimal number + */ + public BigDecimal valueToDecimal(V value) { + if (!this.isDecimal()) { + return null; + } + BigDecimal decimal; + if (value instanceof BigDecimal) { + decimal = (BigDecimal) value; + } else if (value instanceof BigInteger) { + decimal = new BigDecimal((BigInteger) value); + } else if (value instanceof Byte || value instanceof Short || + value instanceof Integer || value instanceof Long) { + decimal = BigDecimal.valueOf(((Number) value).longValue()); + } else if (!(value instanceof Number) && !(value instanceof String)) { + return null; + } else { + String text = value.toString().trim(); + try { + decimal = new BigDecimal(text); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "Can't read '%s' as decimal", value)); + } + } + return checkDecimalBounds(decimal); + } + + public static BigDecimal checkDecimalBounds(BigDecimal decimal) { + int scale = decimal.scale(); + int precision = decimal.precision(); + // Compare the scale directly: Math.abs(Integer.MIN_VALUE) overflows + if (precision > DECIMAL_MAX_PRECISION || + scale < -DECIMAL_MAX_SCALE || scale > DECIMAL_MAX_SCALE) { + throw new IllegalArgumentException(String.format( + "Decimal value out of bounds: precision %d, scale %d " + + "(at most %d significant digits and a scale of at most " + + "%d in either direction)", precision, decimal.scale(), + DECIMAL_MAX_PRECISION, DECIMAL_MAX_SCALE)); + } + return decimal; + } + public Date valueToDate(V value) { if (!this.isDate()) { return null; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/JsonUtil.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/JsonUtil.java index fd00816f66..e9b8890336 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/JsonUtil.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/JsonUtil.java @@ -26,6 +26,7 @@ import org.apache.tinkerpop.shaded.jackson.core.JsonProcessingException; import org.apache.tinkerpop.shaded.jackson.core.type.TypeReference; import org.apache.tinkerpop.shaded.jackson.databind.Module; +import org.apache.tinkerpop.shaded.jackson.databind.DeserializationFeature; import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper; import org.apache.tinkerpop.shaded.jackson.databind.ObjectReader; import org.apache.tinkerpop.shaded.jackson.databind.SerializationFeature; @@ -81,6 +82,25 @@ public static T fromJson(String json, Class clazz) { } } + /** + * Like {@link #fromJson(String, Class)}, but a JSON fraction becomes a + * BigDecimal with every digit instead of a double: for JSON whose + * fractions may be DECIMAL property values (a property key's userdata + * with its default value, the properties filter of the list APIs). + */ + public static T fromJsonExact(String json, Class clazz) { + E.checkState(json != null, + "Json value can't be null for '%s'", + clazz.getSimpleName()); + try { + return MAPPER.readerFor(clazz) + .with(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) + .readValue(json); + } catch (IOException e) { + throw new HugeException("Can't read json: %s", e, e.getMessage()); + } + } + public static T fromJson(String json, TypeReference typeRef) { E.checkState(json != null, "Json value can't be null for '%s'", @@ -113,7 +133,8 @@ public static Object castNumber(Object object, Class clazz) { } else if (clazz == Float.class) { object = number.floatValue(); } else if (clazz == Double.class) { - assert object instanceof Double : object; + // A property value may arrive as BigDecimal (exact JSON fraction) + object = number.doubleValue(); } else { assert clazz == Date.class : clazz; } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/VertexApiTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/VertexApiTest.java index 7321f36d98..0489b21eee 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/VertexApiTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/VertexApiTest.java @@ -18,12 +18,17 @@ package org.apache.hugegraph.api; import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import org.apache.hugegraph.testutil.Assert; import org.junit.Before; import org.junit.Test; import jakarta.ws.rs.core.Response; +import com.google.common.collect.ImmutableMap; + public class VertexApiTest extends BaseApiTest { private static final String PATH = "/graphspaces/DEFAULT/graphs/hugegraph/graph/vertices/"; @@ -98,4 +103,351 @@ public void testDelete() throws IOException { r = client().delete(PATH, id); assertResponseStatus(204, r); } + + @Test + public void testDecimalJsonNumberLiteralIsExact() throws IOException { + createAndAssert(URL_PREFIX + "/schema/propertykeys", + "{" + + "\"name\": \"amount\"," + + "\"data_type\": \"DECIMAL\"," + + "\"cardinality\": \"SINGLE\"," + + "\"check_exist\": false," + + "\"properties\":[]" + + "}", 202); + createAndAssert(URL_PREFIX + "/schema/propertykeys", + "{" + + "\"name\": \"weight\"," + + "\"data_type\": \"DOUBLE\"," + + "\"cardinality\": \"SINGLE\"," + + "\"check_exist\": false," + + "\"properties\":[]" + + "}", 202); + createAndAssert(URL_PREFIX + "/schema/vertexlabels", + "{" + + "\"primary_keys\":[\"name\"]," + + "\"id_strategy\": \"PRIMARY_KEY\"," + + "\"name\": \"transfer\"," + + "\"properties\":[\"name\", \"amount\", \"weight\"]," + + "\"check_exist\": false," + + "\"nullable_keys\":[\"amount\", \"weight\"]" + + "}"); + + // 39 significant digits as a JSON number literal: a double parser + // would keep 17 of them; the value is stored and echoed exactly + String literal = "12345678901234567890.123456789012345678"; + String vertex = "{" + + "\"label\":\"transfer\"," + + "\"properties\":{" + + "\"name\":\"t1\"," + + "\"amount\":" + literal + "," + + "\"weight\":" + literal + "}" + + "}"; + Response r = client().post(PATH, vertex); + String content = assertResponseStatus(201, r); + Assert.assertContains("\"amount\":\"" + literal + "\"", content); + // the DOUBLE key narrows the same literal to a double, as before + Assert.assertContains("\"weight\":1.2345678901234567E19", content); + + r = client().get(PATH, String.format("\"%s\"", parseId(content))); + content = assertResponseStatus(200, r); + Assert.assertContains("\"amount\":\"" + literal + "\"", content); + + // exponent literals are accepted and stored in plain form + vertex = "{" + + "\"label\":\"transfer\"," + + "\"properties\":{" + + "\"name\":\"t2\"," + + "\"amount\":1E-18," + + "\"weight\":2.5}" + + "}"; + r = client().post(PATH, vertex); + content = assertResponseStatus(201, r); + Assert.assertContains("\"amount\":\"0.000000000000000001\"", content); + Assert.assertContains("\"weight\":2.5", content); + + // Filtering by the decimal value is exact on every backend. A vertex + // filter without an index is refused, so the case that reaches the + // store is an edge query by vertex + label + property: on HStore + // the condition is pushed down and must arrive as a BigDecimal + createAndAssert(URL_PREFIX + "/schema/edgelabels", + "{" + + "\"name\": \"pay\"," + + "\"source_label\": \"person\"," + + "\"target_label\": \"person\"," + + "\"frequency\": \"SINGLE\"," + + "\"properties\":[\"amount\"]," + + "\"nullable_keys\":[\"amount\"]," + + "\"check_exist\": false" + + "}"); + String payer = parseId(assertResponseStatus(201, client().post(PATH, + "{\"label\":\"person\",\"properties\":{\"name\":\"payer\"," + + "\"age\":30,\"city\":\"Beijing\"}}"))); + String payee = parseId(assertResponseStatus(201, client().post(PATH, + "{\"label\":\"person\",\"properties\":{\"name\":\"payee\"," + + "\"age\":31,\"city\":\"Beijing\"}}"))); + String edge = "{\"label\":\"pay\",\"outVLabel\":\"person\"," + + "\"inVLabel\":\"person\",\"outV\":\"" + payer + "\"," + + "\"inV\":\"" + payee + "\"," + + "\"properties\":{\"amount\":" + literal + "}}"; + content = assertResponseStatus(201, client().post( + URL_PREFIX + "/graph/edges/", edge)); + Assert.assertContains("\"amount\":\"" + literal + "\"", content); + + String edges = URL_PREFIX + "/graph/edges/"; + r = client().get(edges, ImmutableMap.of( + "vertex_id", id2Json(payer), "direction", "OUT", "label", "pay", + "properties", URLEncoder.encode("{\"amount\":\"" + literal + "\"}", + StandardCharsets.UTF_8))); + content = assertResponseStatus(200, r); + Assert.assertContains("\"amount\":\"" + literal + "\"", content); + String near = literal.substring(0, literal.length() - 1) + "9"; + r = client().get(edges, ImmutableMap.of( + "vertex_id", id2Json(payer), "direction", "OUT", "label", "pay", + "properties", URLEncoder.encode("{\"amount\":\"" + near + "\"}", + StandardCharsets.UTF_8))); + content = assertResponseStatus(200, r); + Assert.assertEquals("{\"edges\":[]}", content); + // the same filter as JSON number literals: parsed exactly, not as a + // double, so the exact value hits and the one-digit change misses + r = client().get(edges, ImmutableMap.of( + "vertex_id", id2Json(payer), "direction", "OUT", "label", "pay", + "properties", URLEncoder.encode("{\"amount\":" + literal + "}", + StandardCharsets.UTF_8))); + content = assertResponseStatus(200, r); + Assert.assertContains("\"amount\":\"" + literal + "\"", content); + r = client().get(edges, ImmutableMap.of( + "vertex_id", id2Json(payer), "direction", "OUT", "label", "pay", + "properties", URLEncoder.encode("{\"amount\":" + near + "}", + StandardCharsets.UTF_8))); + content = assertResponseStatus(200, r); + Assert.assertEquals("{\"edges\":[]}", content); + + // a predicate with a fractional operand is exact as well + r = client().get(edges, ImmutableMap.of( + "vertex_id", id2Json(payer), "direction", "OUT", "label", "pay", + "properties", URLEncoder.encode("{\"amount\":\"P.eq(" + literal + ")\"}", + StandardCharsets.UTF_8))); + content = assertResponseStatus(200, r); + Assert.assertContains("\"amount\":\"" + literal + "\"", content); + r = client().get(edges, ImmutableMap.of( + "vertex_id", id2Json(payer), "direction", "OUT", "label", "pay", + "properties", URLEncoder.encode("{\"amount\":\"P.eq(" + near + ")\"}", + StandardCharsets.UTF_8))); + content = assertResponseStatus(200, r); + Assert.assertEquals("{\"edges\":[]}", content); + r = client().get(edges, ImmutableMap.of( + "vertex_id", id2Json(payer), "direction", "OUT", "label", "pay", + "properties", URLEncoder.encode("{\"amount\":\"P.within(" + near + "," + + literal + ")\"}", StandardCharsets.UTF_8))); + content = assertResponseStatus(200, r); + Assert.assertContains("\"amount\":\"" + literal + "\"", content); + + // a LIST key: a list filter converts every member, a scalar filter + // keeps membership semantics + createAndAssert(URL_PREFIX + "/schema/propertykeys", + "{" + + "\"name\": \"amounts\"," + + "\"data_type\": \"DECIMAL\"," + + "\"cardinality\": \"LIST\"," + + "\"check_exist\": false," + + "\"properties\":[]" + + "}", 202); + createAndAssert(URL_PREFIX + "/schema/edgelabels", + "{" + + "\"name\": \"pays\"," + + "\"source_label\": \"person\"," + + "\"target_label\": \"person\"," + + "\"frequency\": \"SINGLE\"," + + "\"properties\":[\"amounts\"]," + + "\"nullable_keys\":[\"amounts\"]," + + "\"check_exist\": false" + + "}"); + content = assertResponseStatus(201, client().post(URL_PREFIX + "/graph/edges/", + "{\"label\":\"pays\",\"outVLabel\":\"person\"," + + "\"inVLabel\":\"person\",\"outV\":\"" + payer + "\"," + + "\"inV\":\"" + payee + "\"," + + "\"properties\":{\"amounts\":[" + literal + ", 1.0]}}")); + Assert.assertContains("\"amounts\":[\"" + literal + "\",\"1.0\"]", content); + r = client().get(edges, ImmutableMap.of( + "vertex_id", id2Json(payer), "direction", "OUT", "label", "pays", + "properties", URLEncoder.encode("{\"amounts\":[" + literal + ",1.0]}", + StandardCharsets.UTF_8))); + content = assertResponseStatus(200, r); + Assert.assertContains("\"amounts\":[\"" + literal + "\",\"1.0\"]", content); + r = client().get(edges, ImmutableMap.of( + "vertex_id", id2Json(payer), "direction", "OUT", "label", "pays", + "properties", URLEncoder.encode("{\"amounts\":" + near + "}", + StandardCharsets.UTF_8))); + content = assertResponseStatus(200, r); + Assert.assertEquals("{\"edges\":[]}", content); + + // a DECIMAL default value given as a JSON number keeps every digit, + // on create and after the schema is read back from the backend + String fee = "0.1234567890123456789"; + createAndAssert(URL_PREFIX + "/schema/propertykeys", + "{" + + "\"name\": \"fee\"," + + "\"data_type\": \"DECIMAL\"," + + "\"cardinality\": \"SINGLE\"," + + "\"check_exist\": false," + + "\"user_data\": {\"~default_value\": " + fee + "}," + + "\"properties\":[]" + + "}", 202); + r = client().get(URL_PREFIX + "/schema/propertykeys/", "fee"); + content = assertResponseStatus(200, r); + Assert.assertContains("\"~default_value\":\"" + fee + "\"", content); + createAndAssert(URL_PREFIX + "/schema/vertexlabels", + "{" + + "\"primary_keys\":[\"name\"]," + + "\"id_strategy\": \"PRIMARY_KEY\"," + + "\"name\": \"fees\"," + + "\"properties\":[\"name\", \"fee\"]," + + "\"check_exist\": false," + + "\"nullable_keys\":[\"fee\"]" + + "}"); + r = client().post(PATH, "{\"label\":\"fees\",\"properties\":{\"name\":\"f1\"}}"); + content = assertResponseStatus(201, r); + Assert.assertContains("\"fee\":\"" + fee + "\"", content); + + // a fraction elsewhere in a body keeps its usual type: a DOUBLE key + // with a fractional default value round-trips as a JSON number + createAndAssert(URL_PREFIX + "/schema/propertykeys", + "{" + + "\"name\": \"ratio\"," + + "\"data_type\": \"DOUBLE\"," + + "\"cardinality\": \"SINGLE\"," + + "\"check_exist\": false," + + "\"user_data\": {\"~default_value\": 1.5, \"rate\": 0.85}," + + "\"properties\":[]" + + "}", 202); + r = client().get(URL_PREFIX + "/schema/propertykeys/", "ratio"); + content = assertResponseStatus(200, r); + Assert.assertContains("\"~default_value\":1.5", content); + // custom metadata keeps its type: a JSON number, not a string + Assert.assertContains("\"rate\":0.85", content); + // a default of another type keeps the form the user sent (a DATE + // default is converted only when it is applied, as on master) + createAndAssert(URL_PREFIX + "/schema/propertykeys", + "{" + + "\"name\": \"day\"," + + "\"data_type\": \"DATE\"," + + "\"cardinality\": \"SINGLE\"," + + "\"check_exist\": false," + + "\"user_data\": {\"~default_value\": \"2020-01-01\"}," + + "\"properties\":[]" + + "}", 202); + r = client().get(URL_PREFIX + "/schema/propertykeys/", "day"); + content = assertResponseStatus(200, r); + Assert.assertContains("\"~default_value\":\"2020-01-01\"", content); + + // integer keys still reject a fraction, with the usual message + vertex = "{" + + "\"label\":\"person\"," + + "\"properties\":{" + + "\"name\":\"t3\"," + + "\"age\":29.5," + + "\"city\":\"Beijing\"}" + + "}"; + r = client().post(PATH, vertex); + content = assertResponseStatus(400, r); + Assert.assertContains("Invalid property value", content); + } + + @Test + public void testBatchUpdateDecimalWithSumStrategy() throws IOException { + // schema: a decimal balance on an account keyed by name + createAndAssert(URL_PREFIX + "/schema/propertykeys", + "{" + + "\"name\": \"balance\"," + + "\"data_type\": \"DECIMAL\"," + + "\"cardinality\": \"SINGLE\"," + + "\"check_exist\": false," + + "\"properties\":[]" + + "}", 202); + createAndAssert(URL_PREFIX + "/schema/vertexlabels", + "{" + + "\"primary_keys\":[\"name\"]," + + "\"id_strategy\": \"PRIMARY_KEY\"," + + "\"name\": \"account\"," + + "\"properties\":[\"name\", \"balance\"]," + + "\"check_exist\": false," + + "\"nullable_keys\":[\"balance\"]" + + "}"); + + // 2^256 - 2, as a string + String almostMax = "115792089237316195423570985008687907853" + + "269984665640564039457584007913129639934"; + String max = "115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"; + String vertex = "{" + + "\"label\":\"account\"," + + "\"properties\":{" + + "\"name\":\"alice\"," + + "\"balance\":\"" + almostMax + "\"}" + + "}"; + Response r = client().post(PATH, vertex); + String content = assertResponseStatus(201, r); + String id = parseId(content); + Assert.assertContains("\"balance\":\"" + almostMax + "\"", content); + + // SUM through the batch update: the server adds exactly; an integral + // JSON number literal is accepted as the increment + String batch = "{" + + "\"vertices\":[{" + + "\"label\":\"account\"," + + "\"properties\":{" + + "\"name\":\"alice\"," + + "\"balance\":1}" + + "}]," + + "\"update_strategies\":{\"balance\":\"SUM\"}," + + "\"create_if_not_exist\":true" + + "}"; + r = client().put(PATH, "batch", batch, ImmutableMap.of()); + content = assertResponseStatus(200, r); + Assert.assertContains("\"balance\":\"" + max + "\"", content); + + // a fraction as a string and as a JSON number literal (read as + // BigDecimal, see ObjectMapperResolver); two entries for the same + // vertex in one request are combined first, then added to the + // stored value + batch = "{" + + "\"vertices\":[{" + + "\"label\":\"account\"," + + "\"properties\":{" + + "\"name\":\"alice\"," + + "\"balance\":\"0.000000000000000000\"}" + + "},{" + + "\"label\":\"account\"," + + "\"properties\":{" + + "\"name\":\"alice\"," + + "\"balance\":0.000000000000000001}" + + "}]," + + "\"update_strategies\":{\"balance\":\"SUM\"}," + + "\"create_if_not_exist\":true" + + "}"; + r = client().put(PATH, "batch", batch, ImmutableMap.of()); + content = assertResponseStatus(200, r); + String expected = max + ".000000000000000001"; + Assert.assertContains("\"balance\":\"" + expected + "\"", content); + + // read back through GET + r = client().get(PATH, String.format("\"%s\"", id)); + content = assertResponseStatus(200, r); + Assert.assertContains("\"balance\":\"" + expected + "\"", content); + + // BIGGER keeps the larger of the two, compared as decimals + batch = "{" + + "\"vertices\":[{" + + "\"label\":\"account\"," + + "\"properties\":{" + + "\"name\":\"alice\"," + + "\"balance\":\"" + almostMax + "\"}" + + "}]," + + "\"update_strategies\":{\"balance\":\"BIGGER\"}," + + "\"create_if_not_exist\":true" + + "}"; + r = client().put(PATH, "batch", batch, ImmutableMap.of()); + content = assertResponseStatus(200, r); + Assert.assertContains("\"balance\":\"" + expected + "\"", content); + } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeLabelCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeLabelCoreTest.java index 8629f78b3e..0aa45454ce 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeLabelCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeLabelCoreTest.java @@ -1500,4 +1500,34 @@ public void testDuplicateEdgeLabelWithDifferentProperties() { .create(); }); } + + @Test + public void testAddEdgeLabelWithDecimalSortKey() { + super.initPropertyKeys(); + SchemaManager schema = graph().schema(); + schema.propertyKey("amount").asDecimal().create(); + schema.vertexLabel("account") + .properties("name") + .primaryKeys("name") + .create(); + + Assert.assertThrows(IllegalArgumentException.class, () -> { + schema.edgeLabel("transfer").multiTimes() + .properties("amount", "time") + .link("account", "account") + .sortKeys("amount") + .create(); + }, e -> { + Assert.assertContains("can't be a decimal property", + e.getMessage()); + }); + + // a decimal is fine as an ordinary edge property + EdgeLabel transfer = schema.edgeLabel("transfer").multiTimes() + .properties("amount", "time") + .link("account", "account") + .sortKeys("time") + .create(); + Assert.assertEquals(1, transfer.sortKeys().size()); + } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/IndexLabelCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/IndexLabelCoreTest.java index 24a905427c..0e0d377468 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/IndexLabelCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/IndexLabelCoreTest.java @@ -1882,4 +1882,41 @@ public void testDuplicateIndexLabelWithDifferentProperties() { .create(); }); } + + @Test + public void testAddIndexLabelOnDecimalProperty() { + super.initPropertyKeys(); + SchemaManager schema = graph().schema(); + schema.propertyKey("balance").asDecimal().create(); + schema.vertexLabel("account") + .properties("name", "balance") + .primaryKeys("name") + .create(); + + // no byte-order encoding exists for decimals: no index of any type + Assert.assertThrows(IllegalArgumentException.class, () -> { + schema.indexLabel("accountByBalance").onV("account") + .by("balance").secondary().create(); + }, e -> { + Assert.assertContains("data type is decimal", e.getMessage()); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + schema.indexLabel("accountByBalanceRange").onV("account") + .by("balance").range().create(); + }, e -> { + Assert.assertContains("data type is decimal", e.getMessage()); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + schema.indexLabel("accountByNameBalance").onV("account") + .by("name", "balance").shard().create(); + }, e -> { + Assert.assertContains("data type is decimal", e.getMessage()); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + schema.indexLabel("accountByBalanceUnique").onV("account") + .by("balance").unique().create(); + }, e -> { + Assert.assertContains("data type is decimal", e.getMessage()); + }); + } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/PropertyKeyCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/PropertyKeyCoreTest.java index 0609f607ba..c509e74fe6 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/PropertyKeyCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/PropertyKeyCoreTest.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.core; +import java.math.BigDecimal; import java.util.Date; import org.apache.hugegraph.HugeException; @@ -391,6 +392,41 @@ public void testAddOlapPropertyKey() { Assert.assertEquals(WriteType.OLAP_SECONDARY, wcc.writeType()); } + @Test + public void testAddOlapPropertyKeyWithDecimalType() { + Assume.assumeTrue("Not support olap properties", + storeFeatures().supportsOlapProperties()); + + SchemaManager schema = graph().schema(); + + // OLAP_SECONDARY and OLAP_RANGE build an index label on the key + // through SchemaTransaction.createIndexLabelForOlapPk(), which + // skips IndexLabelBuilder.checkFields(): the rule "no index of any + // type on a decimal" has to hold there too + Assert.assertThrows(NotAllowException.class, () -> { + schema.propertyKey("rank").asDecimal().valueSingle() + .writeType(WriteType.OLAP_SECONDARY).create(); + }, e -> { + Assert.assertContains("decimal keys can't be indexed", + e.getMessage()); + }); + Assert.assertThrows(NotAllowException.class, () -> { + schema.propertyKey("rank").asDecimal().valueSingle() + .writeType(WriteType.OLAP_RANGE).create(); + }, e -> { + Assert.assertContains("decimal keys can't be indexed", + e.getMessage()); + }); + Assert.assertFalse(graph().existsIndexLabel("*olap_by_rank")); + + // OLAP_COMMON has no index and stays allowed + PropertyKey rank = schema.propertyKey("rank").asDecimal() + .valueSingle() + .writeType(WriteType.OLAP_COMMON).create(); + Assert.assertEquals(DataType.DECIMAL, rank.dataType()); + Assert.assertEquals(WriteType.OLAP_COMMON, rank.writeType()); + } + @Test public void testClearOlapPropertyKey() { Assume.assumeTrue("Not support olap properties", @@ -741,4 +777,64 @@ public void testDuplicatePropertyKeyWithDifferentProperties() { .create(); }); } + + @Test + public void testAddPropertyKeyWithDecimalType() { + SchemaManager schema = graph().schema(); + PropertyKey balance = schema.propertyKey("balance") + .asDecimal() + .valueSingle() + .create(); + + Assert.assertEquals("balance", balance.name()); + Assert.assertEquals(DataType.DECIMAL, balance.dataType()); + Assert.assertEquals(Cardinality.SINGLE, balance.cardinality()); + Assert.assertEquals(DataType.DECIMAL, + graph().propertyKey("balance").dataType()); + + // values are normalised to BigDecimal, exactly + String uint256Max = "115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"; + Assert.assertEquals(new BigDecimal(uint256Max), + balance.validValue(uint256Max)); + Assert.assertEquals(new BigDecimal("42"), balance.validValue(42L)); + Assert.assertEquals(new BigDecimal("0.1"), balance.validValue(0.1D)); + Assert.assertNull(balance.validValue(true)); + Assert.assertThrows(IllegalArgumentException.class, () -> { + balance.validValue("1,5"); + }, e -> { + Assert.assertContains("Can't read '1,5' as decimal", + e.getMessage()); + }); + // bounds hold for strings and for ready-made BigDecimals alike + Assert.assertThrows(IllegalArgumentException.class, () -> { + balance.validValue("1E+999999999"); + }, e -> { + Assert.assertContains("out of bounds", e.getMessage()); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + balance.validValue(new BigDecimal("1E+999999999")); + }, e -> { + Assert.assertContains("out of bounds", e.getMessage()); + }); + Assert.assertEquals(new BigDecimal("1E+128"), + balance.validValue(new BigDecimal("1E+128"))); + + // SUM/MAX/MIN aggregate types are allowed like on any numeric key + PropertyKey total = schema.propertyKey("total") + .asDecimal() + .calcSum() + .create(); + Assert.assertEquals(AggregateType.SUM, total.aggregateType()); + + // decimal lists and sets + PropertyKey amounts = schema.propertyKey("amounts") + .asDecimal() + .valueList() + .create(); + Assert.assertEquals(Cardinality.LIST, amounts.cardinality()); + Assert.assertEquals(ImmutableList.of(new BigDecimal("1"), + new BigDecimal("2.5")), + amounts.validValue(ImmutableList.of("1", "2.5"))); + } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index 80c4aef50b..630ef9d45d 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.core; +import java.math.BigInteger; import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; @@ -9631,4 +9632,90 @@ private static void assertNotContains(List vertices, Object... keyValues) { Assert.assertFalse(Utils.contains(vertices, new FakeObjects.FakeVertex(keyValues))); } + + @Test + public void testAddVertexWithPropertyValueOfDecimal() { + HugeGraph graph = graph(); + + SchemaManager schema = graph.schema(); + schema.propertyKey("balance").asDecimal().create(); + schema.vertexLabel("account").properties("balance").create(); + + // 2^256 - 1: exact through the write path, the backend and the read path + String uint256Max = "115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"; + BigDecimal expected = new BigDecimal(uint256Max); + Vertex vertex = graph.addVertex(T.label, "account", + "balance", uint256Max); + Assert.assertEquals(expected, vertex.value("balance")); + graph.tx().commit(); + + Vertex loaded = graph.vertex(vertex.id()); + Assert.assertEquals(expected, loaded.value("balance")); + Assert.assertEquals(BigDecimal.class, loaded.value("balance").getClass()); + + // a wei above an ether, in ether: 18 fraction digits kept + BigDecimal wei = new BigDecimal("1.000000000000000001"); + Vertex v2 = graph.addVertex(T.label, "account", "balance", wei); + Vertex v3 = graph.addVertex(T.label, "account", "balance", 42L); + Vertex v4 = graph.addVertex(T.label, "account", + "balance", new BigInteger(uint256Max)); + graph.tx().commit(); + Assert.assertEquals(wei, graph.vertex(v2.id()).value("balance")); + Assert.assertEquals(new BigDecimal("42"), + graph.vertex(v3.id()).value("balance")); + Assert.assertEquals(expected, graph.vertex(v4.id()).value("balance")); + + // equality is exact, not through double: the neighbouring value + // (2^256 - 2) is a different number. Filter by id so no index is + // needed; the condition is evaluated on the server + BigDecimal neighbour = expected.subtract(BigDecimal.ONE); + Object[] ids = {vertex.id(), v2.id(), v3.id(), v4.id()}; + Assert.assertEquals(2L, graph.traversal().V(ids) + .has("balance", expected) + .count().next().longValue()); + Assert.assertEquals(0L, graph.traversal().V(ids) + .has("balance", neighbour) + .count().next().longValue()); + // ranges: 42 and both uint256 values are above 1.000000000000000001 + Assert.assertEquals(3L, graph.traversal().V(ids) + .has("balance", P.gt(wei)) + .count().next().longValue()); + Assert.assertEquals(0L, graph.traversal().V(ids) + .has("balance", P.lt(wei)) + .count().next().longValue()); + Assert.assertEquals(1L, graph.traversal().V(ids) + .has("balance", P.lt(new BigDecimal("42"))) + .count().next().longValue()); + Assert.assertEquals(2L, graph.traversal().V(ids) + .has("balance", P.gte(neighbour)) + .count().next().longValue()); + + // updates keep the exact value + loaded.property("balance", neighbour); + graph.tx().commit(); + Assert.assertEquals(neighbour, graph.vertex(vertex.id()).value("balance")); + } + + @Test + public void testAddVertexWithInvalidPropertyValueOfDecimal() { + HugeGraph graph = graph(); + + SchemaManager schema = graph.schema(); + schema.propertyKey("balance").asDecimal().create(); + schema.vertexLabel("account").properties("balance").create(); + + Assert.assertThrows(IllegalArgumentException.class, () -> { + graph.addVertex(T.label, "account", "balance", "12abc"); + }, e -> { + Assert.assertContains("Can't read '12abc' as decimal", + e.getMessage()); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + graph.addVertex(T.label, "account", "balance", true); + }, e -> { + Assert.assertContains("Invalid property value 'true' " + + "for key 'balance'", e.getMessage()); + }); + } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexLabelCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexLabelCoreTest.java index a43731f235..54045df915 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexLabelCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexLabelCoreTest.java @@ -335,6 +335,42 @@ public void testAddVertexWithPrimaryKeyIdStrategyButNotPassedPk() { }); } + @Test + public void testAddVertexLabelWithDecimalPrimaryKey() { + super.initPropertyKeys(); + SchemaManager schema = graph().schema(); + schema.propertyKey("balance").asDecimal().create(); + + // a decimal cannot be part of the vertex id: LongEncoding/NumericUtil + // collapse fractions into a double and overflow a long on uint256 + Assert.assertThrows(IllegalArgumentException.class, () -> { + schema.vertexLabel("account") + .properties("balance", "name") + .primaryKeys("balance") + .create(); + }, e -> { + Assert.assertContains("can't be a decimal property", + e.getMessage()); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + schema.vertexLabel("account") + .properties("name", "balance") + .primaryKeys("name", "balance") + .create(); + }, e -> { + Assert.assertContains("can't be a decimal property", + e.getMessage()); + }); + Assert.assertFalse(graph().existsVertexLabel("account")); + + // as a plain property next to a text primary key it is fine + VertexLabel account = schema.vertexLabel("account") + .properties("name", "balance") + .primaryKeys("name") + .create(); + Assert.assertEquals(1, account.primaryKeys().size()); + } + @Test public void testAddVertexLabelWith2PrimaryKey() { super.initPropertyKeys(); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index d48738b840..ee642662f2 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -31,6 +31,7 @@ import org.apache.hugegraph.unit.api.auth.LoginAPITest; import org.apache.hugegraph.unit.api.filter.AccessLogFilterTest; import org.apache.hugegraph.unit.api.filter.LoadDetectFilterTest; +import org.apache.hugegraph.unit.api.graph.PropertiesDeserializerTest; import org.apache.hugegraph.unit.api.filter.PathFilterTest; import org.apache.hugegraph.unit.api.gremlin.GremlinQueryAPITest; import org.apache.hugegraph.unit.api.space.GraphSpaceAPITest; @@ -68,6 +69,7 @@ import org.apache.hugegraph.unit.core.ServerInfoManagerTest; import org.apache.hugegraph.unit.core.SystemSchemaStoreTest; import org.apache.hugegraph.unit.core.TaskSchedulerServerInfoTest; +import org.apache.hugegraph.unit.core.PropertyKeyUserdataTest; import org.apache.hugegraph.unit.core.TraversalUtilTest; import org.apache.hugegraph.unit.id.EdgeIdTest; import org.apache.hugegraph.unit.id.IdTest; @@ -81,6 +83,7 @@ import org.apache.hugegraph.unit.serializer.BinaryScatterSerializerTest; import org.apache.hugegraph.unit.serializer.BinarySerializerTest; import org.apache.hugegraph.unit.serializer.BytesBufferTest; +import org.apache.hugegraph.unit.serializer.HugeGraphSONModuleTest; import org.apache.hugegraph.unit.serializer.SerializerFactoryTest; import org.apache.hugegraph.unit.serializer.StoreSerializerTest; import org.apache.hugegraph.unit.serializer.TableBackendEntryTest; @@ -107,6 +110,7 @@ /* api filter */ AccessLogFilterTest.class, LoadDetectFilterTest.class, + PropertiesDeserializerTest.class, LoginAPITest.class, PathFilterTest.class, @@ -165,6 +169,7 @@ HstoreSessionsTest.class, BackendStoreInfoTest.class, TraversalUtilTest.class, + PropertyKeyUserdataTest.class, TraversalUtilOptimizeTest.class, IdHolderTest.class, PageStateTest.class, @@ -189,6 +194,7 @@ BinaryScatterSerializerTest.class, StoreSerializerTest.class, TextSerializerTest.class, + HugeGraphSONModuleTest.class, /* rocksdb */ RocksDBSessionsTest.class, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/api/graph/PropertiesDeserializerTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/api/graph/PropertiesDeserializerTest.java new file mode 100644 index 0000000000..f79cd61459 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/api/graph/PropertiesDeserializerTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.api.graph; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +import org.apache.hugegraph.api.graph.PropertiesDeserializer; +import org.apache.hugegraph.api.schema.UserdataDeserializer; +import org.apache.hugegraph.schema.Userdata; +import org.apache.hugegraph.testutil.Assert; +import org.junit.Test; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +public class PropertiesDeserializerTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + public static class Body { + + @JsonProperty("label") + public String label; + @JsonProperty("properties") + @JsonDeserialize(using = PropertiesDeserializer.class) + public Map properties; + @JsonProperty("options") + public Map options; + } + + @Test + public void testFractionsAreExactInPropertiesOnly() throws Exception { + Body body = MAPPER.readValue( + "{\"label\":\"account\"," + + "\"properties\":{\"amount\":12345678901234567890.123456789012345678," + + "\"rate\":1.10,\"count\":7,\"big\":123456789012345678901234567890," + + "\"name\":\"a\",\"ok\":true,\"none\":null," + + "\"tags\":[1.5,\"x\",[2.25]],\"nested\":{\"w\":0.1}}," + + "\"options\":{\"alpha\":0.85}}", Body.class); + + Map props = body.properties; + Assert.assertEquals(new BigDecimal("12345678901234567890.123456789012345678"), + props.get("amount")); + Assert.assertEquals(new BigDecimal("1.10"), props.get("rate")); + Assert.assertEquals(7, props.get("count")); + Assert.assertEquals(new java.math.BigInteger("123456789012345678901234567890"), + props.get("big")); + Assert.assertEquals("a", props.get("name")); + Assert.assertEquals(Boolean.TRUE, props.get("ok")); + Assert.assertTrue(props.containsKey("none")); + Assert.assertNull(props.get("none")); + List tags = (List) props.get("tags"); + Assert.assertEquals(new BigDecimal("1.5"), tags.get(0)); + Assert.assertEquals("x", tags.get(1)); + Assert.assertEquals(new BigDecimal("2.25"), ((List) tags.get(2)).get(0)); + Assert.assertEquals(new BigDecimal("0.1"), + ((Map) props.get("nested")).get("w")); + // key order is kept + Assert.assertEquals("amount", props.keySet().iterator().next()); + + // everything outside "properties" keeps Jackson's default types + Assert.assertEquals(0.85d, body.options.get("alpha")); + } + + public static class KeyBody { + + @JsonProperty("name") + public String name; + @JsonProperty("user_data") + @JsonDeserialize(using = UserdataDeserializer.class) + public Userdata userdata; + } + + /** The same reading on a property key's user_data (PropertyKeyAPI). */ + @Test + public void testPropertyKeyUserdataIsExact() throws Exception { + KeyBody key = MAPPER.readValue( + "{\"name\":\"fee\"," + + "\"user_data\":{\"~default_value\":0.1234567890123456789,\"note\":\"x\"," + + "\"weight\":2,\"rate\":0.85,\"tags\":[1.5]}}", KeyBody.class); + Assert.assertEquals(new BigDecimal("0.1234567890123456789"), + key.userdata.get("~default_value")); + Assert.assertEquals("x", key.userdata.get("note")); + Assert.assertEquals(2, key.userdata.get("weight")); + // only the default value is exact: other metadata keeps its types + Assert.assertEquals(0.85d, key.userdata.get("rate")); + Assert.assertEquals(1.5d, ((List) key.userdata.get("tags")).get(0)); + key = MAPPER.readValue("{\"name\":\"fee\",\"user_data\":{\"~default_value\":[1.5, 2]}}", + KeyBody.class); + Assert.assertEquals(new BigDecimal("1.5"), + ((List) key.userdata.get("~default_value")).get(0)); + key = MAPPER.readValue("{\"name\":\"fee\"}", KeyBody.class); + Assert.assertNull(key.userdata); + } + + @Test + public void testNullAndNonObject() throws Exception { + Body body = MAPPER.readValue("{\"label\":\"x\",\"properties\":null}", + Body.class); + Assert.assertNull(body.properties); + Assert.assertThrows(com.fasterxml.jackson.databind.JsonMappingException.class, () -> { + MAPPER.readValue("{\"label\":\"x\",\"properties\":[1]}", Body.class); + }); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/ConditionTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/ConditionTest.java index ba4b09dcab..93d9543372 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/ConditionTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/ConditionTest.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.unit.core; +import java.math.BigDecimal; import java.util.Date; import org.apache.hugegraph.backend.id.IdGenerator; @@ -395,6 +396,28 @@ public void testConditionIn() { }); } + @Test + public void testConditionInMatchesNumbersByValue() { + // BigDecimal.equals() is scale sensitive; IN/NOT_IN compare by value + // like EQ, so a stored 1.0 is within (1.00, 2.5) on every backend + Condition c = new SyspropRelation(HugeKeys.ID, RelationType.IN, + ImmutableList.of(new BigDecimal("1.00"), + new BigDecimal("2.5"))); + Assert.assertTrue(c.test(new BigDecimal("1.0"))); + Assert.assertTrue(c.test(new BigDecimal("1"))); + Assert.assertTrue(c.test(1)); + Assert.assertTrue(c.test(2.5d)); + Assert.assertFalse(c.test(new BigDecimal("1.01"))); + Assert.assertFalse(c.test("1.0")); + + Condition n = new SyspropRelation(HugeKeys.ID, RelationType.NOT_IN, + ImmutableList.of(new BigDecimal("1.00"))); + Assert.assertFalse(n.test(new BigDecimal("1.0"))); + Assert.assertFalse(n.test(1L)); + Assert.assertTrue(n.test(new BigDecimal("1.1"))); + Assert.assertTrue(n.test("1.0")); + } + @Test public void testConditionNotIn() { Condition c1 = Condition.nin(HugeKeys.ID, ImmutableList.of(1, 2, "3")); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/DataTypeTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/DataTypeTest.java index 3ccff1a131..5f7c4398fe 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/DataTypeTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/DataTypeTest.java @@ -17,6 +17,8 @@ package org.apache.hugegraph.unit.core; +import java.math.BigInteger; +import java.math.BigDecimal; import java.util.Date; import java.util.UUID; @@ -40,12 +42,15 @@ public void testString() { Assert.assertEquals("blob", DataType.BLOB.string()); Assert.assertEquals("date", DataType.DATE.string()); Assert.assertEquals("uuid", DataType.UUID.string()); + Assert.assertEquals("decimal", DataType.DECIMAL.string()); } @Test public void testValueToNumber() { Assert.assertNull(DataType.BOOLEAN.valueToNumber(1)); Assert.assertNull(DataType.INT.valueToNumber("not number")); + // decimal is not a "number" in the fixed-width sense + Assert.assertNull(DataType.DECIMAL.valueToNumber(1)); Assert.assertEquals((byte) 1, DataType.BYTE.valueToNumber(1)); Assert.assertEquals(1, DataType.INT.valueToNumber(1)); @@ -82,4 +87,120 @@ public void testValueToUUID() { Assert.assertNull(DataType.TEXT.valueToUUID("2019-01-01 12:00:00")); Assert.assertNull(DataType.UUID.valueToUUID(true)); } + + @Test + public void testDecimal() { + Assert.assertTrue(DataType.DECIMAL.isDecimal()); + Assert.assertFalse(DataType.DECIMAL.isNumber()); + Assert.assertFalse(DataType.DECIMAL.isNumber4()); + Assert.assertFalse(DataType.DECIMAL.isNumber8()); + Assert.assertFalse(DataType.DOUBLE.isDecimal()); + Assert.assertEquals(BigDecimal.class, DataType.DECIMAL.clazz()); + Assert.assertEquals(DataType.DECIMAL, + DataType.fromClass(BigDecimal.class)); + } + + @Test + public void testValueToDecimalBounds() { + // a huge exponent is a few bytes on disk and a billion characters + // from toPlainString() on every read: rejected before it is stored + // the extreme scales, where Math.abs(Integer.MIN_VALUE) would overflow + for (int scale : new int[]{Integer.MIN_VALUE, Integer.MIN_VALUE + 1, + Integer.MAX_VALUE}) { + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataType.DECIMAL.valueToDecimal( + new BigDecimal(BigInteger.ONE, scale)); + }, e -> { + Assert.assertContains("out of bounds", e.getMessage()); + }); + } + for (String bad : new String[]{"1E+999999999", "1E-999999999", + "1E+129", "1E-129"}) { + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataType.DECIMAL.valueToDecimal(bad); + }, e -> { + Assert.assertContains("out of bounds", e.getMessage()); + }); + } + // the same check applies to a BigDecimal that arrives ready-made + // (Gremlin literal, SUM result) + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataType.DECIMAL.valueToDecimal(new BigDecimal("1E+999999999")); + }); + // 129 significant digits rejected, 128 accepted + String digits128 = "1".repeat(128); + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataType.DECIMAL.valueToDecimal(digits128 + "1"); + }); + Assert.assertEquals(new BigDecimal(digits128), + DataType.DECIMAL.valueToDecimal(digits128)); + // uint256 max with 18 fraction digits (96 digits) is inside + String uint256Max = "115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"; + BigDecimal wide = new BigDecimal(uint256Max + ".000000000000000001"); + Assert.assertEquals(wide, DataType.DECIMAL.valueToDecimal(wide)); + // scale boundary in both directions + Assert.assertEquals(new BigDecimal("1E+128"), + DataType.DECIMAL.valueToDecimal("1E+128")); + Assert.assertEquals(new BigDecimal("1E-128"), + DataType.DECIMAL.valueToDecimal("1E-128")); + } + + @Test + public void testValueToDecimal() { + // uint256 max: 78 digits, far beyond long and double + String uint256Max = "115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"; + BigDecimal expected = new BigDecimal(uint256Max); + Assert.assertSame(expected, DataType.DECIMAL.valueToDecimal(expected)); + Assert.assertEquals(expected, + DataType.DECIMAL.valueToDecimal(uint256Max)); + Assert.assertEquals(expected, DataType.DECIMAL.valueToDecimal( + new BigInteger(uint256Max))); + Assert.assertEquals(uint256Max, DataType.DECIMAL.valueToDecimal( + " " + uint256Max + " ").toPlainString()); + + // scale is preserved: 1 wei on top of 1 ether, in ether + BigDecimal wei = DataType.DECIMAL.valueToDecimal( + "1.000000000000000001"); + Assert.assertEquals(18, wei.scale()); + Assert.assertEquals("1.000000000000000001", wei.toPlainString()); + + // integral java numbers are exact + Assert.assertEquals(new BigDecimal("42"), + DataType.DECIMAL.valueToDecimal(42)); + Assert.assertEquals(new BigDecimal("42"), + DataType.DECIMAL.valueToDecimal(42L)); + Assert.assertEquals(new BigDecimal("-7"), + DataType.DECIMAL.valueToDecimal((byte) -7)); + // binary floats arrive as their shortest decimal representation + Assert.assertEquals(new BigDecimal("0.1"), + DataType.DECIMAL.valueToDecimal(0.1D)); + Assert.assertEquals(new BigDecimal("1.5"), + DataType.DECIMAL.valueToDecimal(1.5F)); + // negative and zero + Assert.assertEquals(new BigDecimal("-0.5"), + DataType.DECIMAL.valueToDecimal("-0.5")); + Assert.assertEquals(BigDecimal.ZERO, + DataType.DECIMAL.valueToDecimal("0")); + + // not convertible + Assert.assertNull(DataType.DECIMAL.valueToDecimal(true)); + Assert.assertNull(DataType.DECIMAL.valueToDecimal(new Date())); + Assert.assertNull(DataType.TEXT.valueToDecimal("1.5")); + Assert.assertNull(DataType.DOUBLE.valueToDecimal(1.5D)); + + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataType.DECIMAL.valueToDecimal("12abc"); + }, e -> { + Assert.assertContains("Can't read '12abc' as decimal", + e.getMessage()); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataType.DECIMAL.valueToDecimal(""); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + DataType.DECIMAL.valueToDecimal("0x10"); + }); + } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/PropertyKeyUserdataTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/PropertyKeyUserdataTest.java new file mode 100644 index 0000000000..6c4790902b --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/PropertyKeyUserdataTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.core; + +import java.math.BigDecimal; +import java.util.Arrays; + +import org.apache.hugegraph.backend.id.IdGenerator; +import org.apache.hugegraph.schema.PropertyKey; +import org.apache.hugegraph.schema.Userdata; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.type.define.Cardinality; +import org.apache.hugegraph.type.define.DataType; +import org.junit.Test; + +/** + * Only a DECIMAL key normalizes its ~default_value eagerly; every other type + * keeps the raw value the user sent (and an unconvertible stored default + * still loads), exactly as on master. + */ +public class PropertyKeyUserdataTest { + + private static PropertyKey key(DataType type, Cardinality cardinality) { + PropertyKey pkey = new PropertyKey(null, IdGenerator.of(1), "k"); + pkey.dataType(type); + pkey.cardinality(cardinality); + return pkey; + } + + @Test + public void testDecimalDefaultIsExact() { + PropertyKey pkey = key(DataType.DECIMAL, Cardinality.SINGLE); + pkey.userdata(Userdata.DEFAULT_VALUE, new BigDecimal("0.1234567890123456789")); + Assert.assertEquals(new BigDecimal("0.1234567890123456789"), + pkey.userdata().get(Userdata.DEFAULT_VALUE)); + pkey.userdata(Userdata.DEFAULT_VALUE, "1.50"); + Assert.assertEquals(new BigDecimal("1.50"), pkey.userdata().get(Userdata.DEFAULT_VALUE)); + PropertyKey list = key(DataType.DECIMAL, Cardinality.LIST); + list.userdata(Userdata.DEFAULT_VALUE, Arrays.asList(new BigDecimal("1.5"), 2)); + Assert.assertEquals(Arrays.asList(new BigDecimal("1.5"), new BigDecimal("2")), + list.userdata().get(Userdata.DEFAULT_VALUE)); + // an unconvertible default is kept raw instead of failing the load + pkey.userdata(Userdata.DEFAULT_VALUE, "not-a-number"); + Assert.assertEquals("not-a-number", pkey.userdata().get(Userdata.DEFAULT_VALUE)); + } + + @Test + public void testOtherTypesKeepTheRawDefault() { + PropertyKey date = key(DataType.DATE, Cardinality.SINGLE); + date.userdata(Userdata.DEFAULT_VALUE, "2020-01-01"); + Assert.assertEquals("2020-01-01", date.userdata().get(Userdata.DEFAULT_VALUE)); + date.userdata(Userdata.DEFAULT_VALUE, "not-a-date"); + Assert.assertEquals("not-a-date", date.userdata().get(Userdata.DEFAULT_VALUE)); + + PropertyKey ints = key(DataType.INT, Cardinality.LIST); + ints.userdata(Userdata.DEFAULT_VALUE, 1); + Assert.assertEquals(1, ints.userdata().get(Userdata.DEFAULT_VALUE)); + + PropertyKey dbl = key(DataType.DOUBLE, Cardinality.SINGLE); + Userdata userdata = new Userdata(); + userdata.put(Userdata.DEFAULT_VALUE, 1.5d); + userdata.put("rate", 0.85d); + dbl.userdata(userdata); + Assert.assertEquals(1.5d, dbl.userdata().get(Userdata.DEFAULT_VALUE)); + Assert.assertEquals(0.85d, dbl.userdata().get("rate")); + // the API reads the default exactly; for a non-decimal key that + // BigDecimal becomes the Double the parser gave on master + dbl.userdata(Userdata.DEFAULT_VALUE, new BigDecimal("1.5")); + Assert.assertEquals(1.5d, dbl.userdata().get(Userdata.DEFAULT_VALUE)); + PropertyKey dbls = key(DataType.DOUBLE, Cardinality.LIST); + dbls.userdata(Userdata.DEFAULT_VALUE, Arrays.asList(new BigDecimal("1.5"), 2)); + Assert.assertEquals(Arrays.asList(1.5d, 2), dbls.userdata().get(Userdata.DEFAULT_VALUE)); + ints.userdata(Userdata.DEFAULT_VALUE, new BigDecimal("1.5")); + Assert.assertEquals(1.5d, ints.userdata().get(Userdata.DEFAULT_VALUE)); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/TraversalUtilTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/TraversalUtilTest.java index 08322112b7..923c394a65 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/TraversalUtilTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/TraversalUtilTest.java @@ -17,6 +17,8 @@ package org.apache.hugegraph.unit.core; +import java.math.BigDecimal; + import org.apache.hugegraph.HugeException; import org.apache.hugegraph.testutil.Assert; import org.apache.hugegraph.traversal.optimize.TraversalUtil; @@ -46,44 +48,44 @@ public void testParsePredicate() { TraversalUtil.parsePredicate("P.lt(-1)")); Assert.assertEquals(P.lt(-1), TraversalUtil.parsePredicate("P.lt(\"-1\")")); - Assert.assertEquals(P.lte(-123.45), + Assert.assertEquals(P.lte(new BigDecimal("-123.45")), TraversalUtil.parsePredicate("P.lte(-123.45)")); - Assert.assertEquals(P.lte(3.14), + Assert.assertEquals(P.lte(new BigDecimal("3.14")), TraversalUtil.parsePredicate("P.lte(\"3.14\")")); Assert.assertEquals(P.gt(18), TraversalUtil.parsePredicate("P.gt(18)")); Assert.assertEquals(P.gt(18), TraversalUtil.parsePredicate("P.gt(\"18\")")); - Assert.assertEquals(P.gte(3.14), + Assert.assertEquals(P.gte(new BigDecimal("3.14")), TraversalUtil.parsePredicate("P.gte(3.14)")); - Assert.assertEquals(P.gte(3.14), + Assert.assertEquals(P.gte(new BigDecimal("3.14")), TraversalUtil.parsePredicate("P.gte(\"3.14\")")); Assert.assertEquals(P.between(1, 100), TraversalUtil.parsePredicate("P.between(1, 100)")); - Assert.assertEquals(P.between(1, 1.2), + Assert.assertEquals(P.between(1, new BigDecimal("1.2")), TraversalUtil.parsePredicate("P.between(1, 1.2)")); Assert.assertEquals(P.between(1, 2), TraversalUtil.parsePredicate("P.between(\"1\", 2)")); Assert.assertEquals(P.inside(1, 100), TraversalUtil.parsePredicate("P.inside(1, 100)")); - Assert.assertEquals(P.inside(0.28, 1), + Assert.assertEquals(P.inside(new BigDecimal("0.28"), 1), TraversalUtil.parsePredicate("P.inside(0.28, 1)")); Assert.assertEquals(P.inside(1, 2), TraversalUtil.parsePredicate("P.inside(\"1\", 2)")); Assert.assertEquals(P.outside(1, 100), TraversalUtil.parsePredicate("P.outside(1, 100)")); - Assert.assertEquals(P.outside(1, 1.5), + Assert.assertEquals(P.outside(1, new BigDecimal("1.5")), TraversalUtil.parsePredicate("P.outside(1, 1.5)")); Assert.assertEquals(P.outside(1, 2), TraversalUtil.parsePredicate("P.outside(\"1\", 2)")); Assert.assertEquals(P.within(1, 3, 5), TraversalUtil.parsePredicate("P.within(1, 3, 5)")); - Assert.assertEquals(P.within("abc", "hello", (Object) 123, 3.14), + Assert.assertEquals(P.within("abc", "hello", (Object) 123, new BigDecimal("3.14")), TraversalUtil.parsePredicate( "P.within(\"abc\", \"hello\", 123, 3.14)")); } @@ -335,4 +337,19 @@ public void testParsePredicateWithInvalidString() { "expect a list", e.getMessage()); }); } + + /** A fractional operand keeps every digit; the key's type converts it later. */ + @Test + public void testParsePredicateFractionIsExact() { + Assert.assertEquals(P.eq(new BigDecimal("0.100000000000000001")), + TraversalUtil.parsePredicate("P.eq(0.100000000000000001)")); + Assert.assertEquals(P.gt(new BigDecimal("12345678901234567890.123456789012345678")), + TraversalUtil.parsePredicate( + "P.gt(12345678901234567890.123456789012345678)")); + Assert.assertEquals(P.between(new BigDecimal("0.1"), new BigDecimal("0.2")), + TraversalUtil.parsePredicate("P.between(0.1, 0.2)")); + Assert.assertEquals(P.within(new BigDecimal("1.00"), 2, "x"), + TraversalUtil.parsePredicate("P.within(1.00, 2, \"x\")")); + Assert.assertEquals(P.eq(7), TraversalUtil.parsePredicate("P.eq(7)")); + } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java index 8d82a7c6c8..0ab08f495d 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.unit.serializer; +import java.math.BigDecimal; import java.awt.Point; import java.lang.reflect.Field; import java.util.Arrays; @@ -1024,6 +1025,36 @@ public void testProperty() { Assert.assertArrayEquals(bytes, buf.writeProperty(pkey, value).bytes()); Assert.assertEquals(value, BytesBuffer.wrap(bytes).readProperty(pkey)); + // decimal = vint(len) + two's-complement unscaled bytes + vint(scale) + pkey = genPkey(DataType.DECIMAL); + value = new BigDecimal("-1.5"); // unscaled -15 (0xf1), scale 1 + bytes = genBytes("01f101"); + buf.forReadWritten(); + Assert.assertArrayEquals(bytes, buf.writeProperty(pkey, value).bytes()); + Assert.assertEquals(value, BytesBuffer.wrap(bytes).readProperty(pkey)); + + value = BigDecimal.ZERO; + bytes = genBytes("010000"); + buf.forReadWritten(); + Assert.assertArrayEquals(bytes, buf.writeProperty(pkey, value).bytes()); + Assert.assertEquals(value, BytesBuffer.wrap(bytes).readProperty(pkey)); + + // uint256 max: 33 bytes (sign byte + 32 × 0xff), scale 0 + value = new BigDecimal("115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"); + bytes = genBytes("2100" + "ff".repeat(32) + "00"); + buf.forReadWritten(); + Assert.assertArrayEquals(bytes, buf.writeProperty(pkey, value).bytes()); + Assert.assertEquals(value, BytesBuffer.wrap(bytes).readProperty(pkey)); + + // scale survives the round trip (1 wei above 1 ether, in ether) + value = new BigDecimal("1.000000000000000001"); + buf.forReadWritten(); + bytes = buf.writeProperty(pkey, value).bytes(); + Object read = BytesBuffer.wrap(bytes).readProperty(pkey); + Assert.assertEquals(value, read); + Assert.assertEquals(18, ((BigDecimal) read).scale()); + pkey = genPkey(DataType.OBJECT); value = new Point(3, 8); bytes = genBytes("1301006a6176612e6177742e506f696ef4010610"); @@ -1133,6 +1164,13 @@ public void testPropertyWithList() { Assert.assertArrayEquals(bytes, buf.writeProperty(pkey, value).bytes()); Assert.assertEquals(value, BytesBuffer.wrap(bytes).readProperty(pkey)); + pkey = genListPkey(DataType.DECIMAL); + value = ImmutableList.of(new BigDecimal("0.1"), new BigDecimal("1e21"), + new BigDecimal("-0.000000000000000001")); + buf.forReadWritten(); + bytes = buf.writeProperty(pkey, value).bytes(); + Assert.assertEquals(value, BytesBuffer.wrap(bytes).readProperty(pkey)); + pkey = genListPkey(DataType.OBJECT); value = ImmutableList.of(new Point(3, 8), new Point(3, 9)); bytes = genBytes("021301006a6176612e6177742e506f696ef4010610" + diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/HugeGraphSONModuleTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/HugeGraphSONModuleTest.java new file mode 100644 index 0000000000..ea01f3f432 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/HugeGraphSONModuleTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.serializer; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.hugegraph.io.HugeGraphIoRegistry; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.unit.BaseUnitTest; +import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage; +import org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0; +import org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0; +import org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0; +import org.apache.tinkerpop.gremlin.driver.ser.MessageTextSerializer; +import org.apache.tinkerpop.gremlin.driver.ser.SerializationException; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +/** + * The module is registered into the gremlin-server GraphSON mappers through + * HugeGraphIoRegistry (gremlin-server.yaml: ioRegistries). The typed + * mappers (v2/v3) call serializeWithType(), so every serializer added by + * the module has to implement it or Gremlin results of that type fail. + */ +public class HugeGraphSONModuleTest extends BaseUnitTest { + + private static final Map CONFIG = ImmutableMap.of( + "ioRegistries", + ImmutableList.of(HugeGraphIoRegistry.class.getName())); + + private static final BigDecimal DECIMAL = new BigDecimal("1.5"); + private static final BigDecimal WEI = new BigDecimal( + "0.000000000000000001"); + private static final BigDecimal UINT256_MAX = new BigDecimal( + "115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"); + + private static ResponseMessage response(Object... results) { + return ResponseMessage.build(UUID.randomUUID()) + .result(ImmutableList.copyOf(results)) + .create(); + } + + private static Object firstResult(ResponseMessage message) { + @SuppressWarnings("unchecked") + List data = (List) message.getResult().getData(); + return data.get(0); + } + + @Test + public void testBigDecimalThroughGraphSONV1() throws Exception { + GraphSONMessageSerializerV1d0 serializer = + new GraphSONMessageSerializerV1d0(); + serializer.configure(CONFIG, null); + + String json = serializer.serializeResponseAsString(response(DECIMAL)); + Assert.assertContains("\"1.5\"", json); + } + + @Test + public void testBigDecimalThroughGraphSONV2() throws Exception { + GraphSONMessageSerializerV2d0 serializer = + new GraphSONMessageSerializerV2d0(); + serializer.configure(CONFIG, null); + this.assertTypedRoundTrip(serializer); + } + + @Test + public void testBigDecimalThroughGraphSONV3() throws Exception { + GraphSONMessageSerializerV3d0 serializer = + new GraphSONMessageSerializerV3d0(); + serializer.configure(CONFIG, null); + this.assertTypedRoundTrip(serializer); + } + + private void assertTypedRoundTrip(MessageTextSerializer serializer) + throws SerializationException { + for (BigDecimal value : ImmutableList.of(DECIMAL, WEI, UINT256_MAX)) { + String json = serializer.serializeResponseAsString( + response(value)); + // the type prefix survives, the value travels as a plain string + Assert.assertContains("gx:BigDecimal", json); + Assert.assertContains("\"" + value.toPlainString() + "\"", json); + + ResponseMessage read = serializer.deserializeResponse(json); + Object result = firstResult(read); + Assert.assertEquals(BigDecimal.class, result.getClass()); + Assert.assertEquals(value, result); + } + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/StoreSerializerTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/StoreSerializerTest.java index 321f018de2..82682279f8 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/StoreSerializerTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/StoreSerializerTest.java @@ -17,8 +17,13 @@ package org.apache.hugegraph.unit.serializer; +import java.math.BigDecimal; import java.util.Iterator; +import org.apache.hugegraph.backend.id.Id; +import org.apache.hugegraph.backend.id.IdGenerator; +import org.apache.hugegraph.backend.query.Condition; +import org.apache.hugegraph.backend.query.ConditionQuery; import org.apache.hugegraph.backend.serializer.BinaryBackendEntry; import org.apache.hugegraph.backend.serializer.BytesBuffer; import org.apache.hugegraph.backend.store.BackendAction; @@ -33,6 +38,8 @@ import org.apache.hugegraph.type.define.Action; import org.junit.Test; +import com.google.common.collect.ImmutableList; + public class StoreSerializerTest { @Test @@ -63,6 +70,35 @@ public void testSerializeBackendMutation() { } } + @Test + public void testConditionQueryBytesKeepBigDecimal() { + // A DECIMAL filter travels to the store as a BigDecimal, not as an + // untyped JSON number that Gson would read back as a double + Id key = IdGenerator.of(7L); + BigDecimal exact = new BigDecimal("12345678901234567890.123456789012345678"); + ConditionQuery query = new ConditionQuery(HugeType.VERTEX); + query.query(Condition.eq(key, exact)); + query.query(Condition.gte(IdGenerator.of(8L), new BigDecimal("0.000000000000000001"))); + query.query(Condition.in(IdGenerator.of(9L), + ImmutableList.of(new BigDecimal("1.10"), + new BigDecimal("2.20")))); + + ConditionQuery copy = ConditionQuery.fromBytes(query.bytes()); + Object value = copy.userpropValue(key); + Assert.assertEquals(BigDecimal.class, value.getClass()); + Assert.assertEquals(exact, value); + Object low = copy.userpropValue(IdGenerator.of(8L)); + Assert.assertEquals(new BigDecimal("0.000000000000000001"), low); + Object list = copy.userpropValue(IdGenerator.of(9L)); + Assert.assertEquals(ImmutableList.of(new BigDecimal("1.10"), + new BigDecimal("2.20")), list); + // a plain double condition is unchanged + query = new ConditionQuery(HugeType.VERTEX); + query.query(Condition.eq(key, 1.5d)); + Assert.assertEquals(1.5d, ConditionQuery.fromBytes(query.bytes()) + .userpropValue(key)); + } + @Test public void testSerializeStoreCommand() { BinaryBackendEntry entry = new BinaryBackendEntry(HugeType.VERTEX, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/util/JsonUtilTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/util/JsonUtilTest.java index c76d536ff0..42e9bfeff1 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/util/JsonUtilTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/util/JsonUtilTest.java @@ -17,6 +17,8 @@ package org.apache.hugegraph.unit.util; +import java.math.BigDecimal; +import java.util.Map; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -54,6 +56,8 @@ import org.apache.tinkerpop.shaded.jackson.core.type.TypeReference; import org.eclipse.collections.api.map.primitive.MutableIntObjectMap; import org.junit.Test; + +import com.google.common.collect.ImmutableMap; import org.mockito.Mockito; import com.google.common.collect.ImmutableList; @@ -316,4 +320,50 @@ public void testDeserializeList() { Assert.assertEquals(ImmutableList.of(1, 2, 3), JsonUtil.fromJson(json, typeRef)); } + + @Test + public void testSerializeBigDecimal() { + // decimals travel as plain strings, never as JSON numbers + BigDecimal decimal = new BigDecimal("1e21"); + Assert.assertEquals("\"1000000000000000000000\"", + JsonUtil.toJson(decimal)); + Assert.assertEquals("\"0.000000000000000001\"", + JsonUtil.toJson(new BigDecimal("1E-18"))); + Assert.assertEquals("\"-1.50\"", + JsonUtil.toJson(new BigDecimal("-1.50"))); + Assert.assertEquals("{\"balance\":\"1000000000000000000000\"}", + JsonUtil.toJson(ImmutableMap.of("balance", decimal))); + // beyond the DECIMAL scale bound (a generic Gremlin result, not a + // property) the scientific form: exact, and no exponent expansion + Assert.assertEquals("\"1E+999999999\"", + JsonUtil.toJson(new BigDecimal("1E+999999999"))); + Assert.assertEquals("\"1E-129\"", + JsonUtil.toJson(new BigDecimal("1E-129"))); + String zeros128 = new String(new char[128]).replace("\0", "0"); + Assert.assertEquals("\"1" + zeros128 + "\"", + JsonUtil.toJson(new BigDecimal("1E+128"))); + // a double stays a JSON number: job parameters and schema userdata + // are not touched by the decimal handling + Assert.assertEquals("{\"alpha\":0.85}", + JsonUtil.toJson(ImmutableMap.of("alpha", 0.85d))); + Assert.assertEquals(0.85d, JsonUtil.fromJson("{\"alpha\":0.85}", + Map.class).get("alpha")); + + // fromJsonExact: a fraction in an untyped map keeps every digit + Map exact = JsonUtil.fromJsonExact( + "{\"amount\":12345678901234567890.123456789012345678,\"n\":7}", Map.class); + Assert.assertEquals(new BigDecimal("12345678901234567890.123456789012345678"), + exact.get("amount")); + Assert.assertEquals(7, exact.get("n")); + Assert.assertEquals(0.1d, JsonUtil.fromJson("{\"w\":0.1}", Map.class).get("w")); + + // both a string and a number literal are accepted on the way in + Assert.assertEquals(new BigDecimal("1.5"), + JsonUtil.fromJson("\"1.5\"", BigDecimal.class)); + Assert.assertEquals(new BigDecimal("1.5"), + JsonUtil.fromJson("1.5", BigDecimal.class)); + Assert.assertEquals(new BigDecimal("1000000000000000000000"), + JsonUtil.fromJson("1000000000000000000000", + BigDecimal.class)); + } } diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/GraphStoreIterator.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/GraphStoreIterator.java index 51b9c8d15c..4d1d1e991e 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/GraphStoreIterator.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/GraphStoreIterator.java @@ -17,7 +17,9 @@ package org.apache.hugegraph.store.business; +import java.math.BigDecimal; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashSet; import java.util.Iterator; @@ -255,6 +257,12 @@ private

> List buildProperties( variant.setType(VariantType.VT_DOUBLE) .setValueDouble((Double) v); break; + case DECIMAL: + // A LIST/SET value is a collection of decimals: every + // member in plain form, as a JSON array of strings + variant.setType(VariantType.VT_STRING) + .setValueString(decimalString(v)); + break; case OBJECT: case UNKNOWN: variant.setType(VariantType.VT_UNKNOWN) @@ -273,6 +281,20 @@ private

> List buildProperties( return props; } + private static String decimalString(Object value) { + if (!(value instanceof Collection)) { + return ((BigDecimal) value).toPlainString(); + } + StringBuilder sb = new StringBuilder("["); + for (Object member : (Collection) value) { + if (sb.length() > 1) { + sb.append(','); + } + sb.append('"').append(((BigDecimal) member).toPlainString()).append('"'); + } + return sb.append(']').toString(); + } + private void buildId(Builder variant, Id id) { switch (id.type()) { case STRING: diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Condition.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Condition.java index 415405a1bd..5fa0b71c0b 100644 --- a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Condition.java +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Condition.java @@ -31,6 +31,7 @@ import org.apache.hugegraph.util.E; import org.apache.hugegraph.util.NumericUtil; +import java.math.BigDecimal; import java.util.*; import java.util.function.BiFunction; import java.util.function.BiPredicate; @@ -225,12 +226,12 @@ public enum RelationType implements BiPredicate { IN("in", null, Collection.class, (v1, v2) -> { assert v2 != null; - return ((Collection) v2).contains(v1); + return collectionContains((Collection) v2, v1); }), NOT_IN("notin", null, Collection.class, (v1, v2) -> { assert v2 != null; - return !((Collection) v2).contains(v1); + return !collectionContains((Collection) v2, v1); }), PREFIX("prefix", Id.class, Id.class, (v1, v2) -> { @@ -568,6 +569,29 @@ public boolean isFuzzyType() { this == TEXT_MATCH_EDIT_DISTANCE; } + /** + * Membership by value once a BigDecimal is involved: its equals() is + * scale sensitive (1.0 vs 1.00), while EQ compares decimals by value, + * so IN/NOT_IN do the same. Other numbers keep contains() semantics + * (an Integer 1 is not a Double 1.0 here, as before). + */ + private static boolean collectionContains(Collection values, Object value) { + if (values.contains(value)) { + return true; + } + if (!(value instanceof Number)) { + return false; + } + boolean decimal = value instanceof BigDecimal; + for (Object member : values) { + if (member instanceof Number && (decimal || member instanceof BigDecimal) && + NumericUtil.compareNumber(value, (Number) member) == 0) { + return true; + } + } + return false; + } + public boolean isRangeType() { return ImmutableSet.of(GT, GTE, LT, LTE).contains(this); } diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/ConditionQuery.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/ConditionQuery.java index 553fec9b8c..b56e9dd8ce 100644 --- a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/ConditionQuery.java +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/ConditionQuery.java @@ -171,6 +171,11 @@ private static boolean numberEquals(Object number1, Object number2) { // Otherwise convert to BigDecimal to make two numbers comparable Number n1 = NumericUtil.convertToNumber(number1); Number n2 = NumericUtil.convertToNumber(number2); + if (n1 instanceof BigDecimal || n2 instanceof BigDecimal) { + // Exact: a decimal must not be squeezed through a double + return new BigDecimal(n1.toString()) + .compareTo(new BigDecimal(n2.toString())) == 0; + } BigDecimal b1 = BigDecimal.valueOf(n1.doubleValue()); BigDecimal b2 = BigDecimal.valueOf(n2.doubleValue()); return b1.compareTo(b2) == 0; @@ -1036,6 +1041,11 @@ private static boolean numberEquals(Object number1, Object number2) { // Otherwise convert to BigDecimal to make two numbers comparable Number n1 = NumericUtil.convertToNumber(number1); Number n2 = NumericUtil.convertToNumber(number2); + if (n1 instanceof BigDecimal || n2 instanceof BigDecimal) { + // Exact: a decimal must not be squeezed through a double + return new BigDecimal(n1.toString()) + .compareTo(new BigDecimal(n2.toString())) == 0; + } BigDecimal b1 = BigDecimal.valueOf(n1.doubleValue()); BigDecimal b2 = BigDecimal.valueOf(n2.doubleValue()); return b1.compareTo(b2) == 0; diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/QueryAdapter.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/QueryAdapter.java index e9975f57cd..bc6d043c9d 100644 --- a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/QueryAdapter.java +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/QueryAdapter.java @@ -18,6 +18,8 @@ package org.apache.hugegraph.query.serializer; import java.lang.reflect.Type; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -48,8 +50,13 @@ public class QueryAdapter extends AbstractSerializerAdapter { .build(); static boolean isPrimitive(Class clz) { + // Values whose class must travel with them: Gson reads an untyped + // JSON number back as a double, which would round a BigDecimal + if (clz == Date.class || clz == BigDecimal.class || clz == BigInteger.class) { + return true; + } try { - return (clz == Date.class) || ((Class) clz.getField("TYPE").get(null)).isPrimitive(); + return ((Class) clz.getField("TYPE").get(null)).isPrimitive(); } catch (Exception e) { return false; } diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/BytesBuffer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/BytesBuffer.java index 4ec6aad194..5cb8e0293b 100644 --- a/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/BytesBuffer.java +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/BytesBuffer.java @@ -19,6 +19,8 @@ package org.apache.hugegraph.serializer; +import java.math.BigDecimal; +import java.math.BigInteger; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -643,6 +645,13 @@ public void writeProperty(DataType dataType, Object value) { this.writeLong(uuid.getMostSignificantBits()); this.writeLong(uuid.getLeastSignificantBits()); break; + case DECIMAL: + // unscaled two's-complement bytes + scale: exact for any + // precision, 33 bytes for a 78-digit (uint256) value + BigDecimal decimal = (BigDecimal) value; + this.writeBytes(decimal.unscaledValue().toByteArray()); + this.writeVInt(decimal.scale()); + break; default: throw new IllegalArgumentException("Unsupported data type " + dataType); } @@ -670,6 +679,9 @@ public Object readProperty(DataType dataType) { return Blob.wrap(this.readBigBytes()); case UUID: return new UUID(this.readLong(), this.readLong()); + case DECIMAL: + BigInteger unscaled = new BigInteger(this.readBytes()); + return new BigDecimal(unscaled, this.readVInt()); default: throw new IllegalArgumentException("Unsupported data type " + dataType); } diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/struct/schema/PropertyKey.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/struct/schema/PropertyKey.java index 81dae36697..b1632dc0a5 100644 --- a/hugegraph-struct/src/main/java/org/apache/hugegraph/struct/schema/PropertyKey.java +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/struct/schema/PropertyKey.java @@ -316,8 +316,11 @@ private V convValue(V value) { if (value == null) { return null; } - if (this.checkValueType(value)) { - // Same as expected type, no conversion required + if (this.checkValueType(value) && !this.dataType().isDecimal()) { + // Same as expected type, no conversion required. A decimal is + // not short-circuited: a ready-made BigDecimal (Gremlin literal, + // SUM result of a batch update) still has to pass the bounds + // check in DataType.valueToDecimal() return value; } @@ -373,6 +376,10 @@ private V convSingleValue(V value) { @SuppressWarnings("unchecked") V blob = (V) this.dataType().valueToBlob(value); return blob; + } else if (this.dataType().isDecimal()) { + @SuppressWarnings("unchecked") + V decimal = (V) this.dataType().valueToDecimal(value); + return decimal; } if (this.checkDataType(value)) { @@ -424,6 +431,9 @@ public String convert2Groovy(boolean attachIdFlag) { case UUID: builder.append(".asUUID()"); break; + case DECIMAL: + builder.append(".asDecimal()"); + break; case OBJECT: builder.append(".asObject()"); break; @@ -548,6 +558,8 @@ public interface Builder extends SchemaBuilder { Builder asLong(); + Builder asDecimal(); + Builder valueSingle(); Builder valueList(); diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/DataType.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/DataType.java index 6a04a83034..6f2f4bd547 100644 --- a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/DataType.java +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/DataType.java @@ -19,6 +19,8 @@ package org.apache.hugegraph.type.define; +import java.math.BigDecimal; +import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.Date; import java.util.List; @@ -46,7 +48,12 @@ public enum DataType implements SerialEnum { TEXT(8, "text", String.class), BLOB(9, "blob", Blob.class), DATE(10, "date", Date.class), - UUID(11, "uuid", UUID.class); + UUID(11, "uuid", UUID.class), + /* + * Arbitrary-precision decimal (java.math.BigDecimal), see the server copy + * of this enum: exact, but not a sort key / index / OLAP range type. + */ + DECIMAL(12, "decimal", BigDecimal.class); private final byte code; private final String name; @@ -109,6 +116,10 @@ public boolean isUUID() { return this == DataType.UUID; } + public boolean isDecimal() { + return this == DataType.DECIMAL; + } + public Number valueToNumber(V value) { if (!(this.isNumber() && value instanceof Number) && !(value instanceof String && SPECIAL_FLOATS.contains(value))) { @@ -149,6 +160,57 @@ public Number valueToNumber(V value) { return number; } + /* + * Bounds for a DECIMAL value: at most DECIMAL_MAX_PRECISION significant + * digits and an absolute scale of at most DECIMAL_MAX_SCALE. uint256 + * with 18 fraction digits is 96 digits, so both fit with room to spare, + * while "1E+999999999" (a few bytes on disk, a billion characters from + * toPlainString() on every read) is rejected before it is stored. + */ + public static final int DECIMAL_MAX_PRECISION = 128; + public static final int DECIMAL_MAX_SCALE = 128; + + public BigDecimal valueToDecimal(V value) { + if (!this.isDecimal()) { + return null; + } + BigDecimal decimal; + if (value instanceof BigDecimal) { + decimal = (BigDecimal) value; + } else if (value instanceof BigInteger) { + decimal = new BigDecimal((BigInteger) value); + } else if (value instanceof Byte || value instanceof Short || + value instanceof Integer || value instanceof Long) { + decimal = BigDecimal.valueOf(((Number) value).longValue()); + } else if (!(value instanceof Number) && !(value instanceof String)) { + return null; + } else { + String text = value.toString().trim(); + try { + decimal = new BigDecimal(text); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "Can't read '%s' as decimal", value)); + } + } + return checkDecimalBounds(decimal); + } + + public static BigDecimal checkDecimalBounds(BigDecimal decimal) { + int scale = decimal.scale(); + int precision = decimal.precision(); + // Compare the scale directly: Math.abs(Integer.MIN_VALUE) overflows + if (precision > DECIMAL_MAX_PRECISION || + scale < -DECIMAL_MAX_SCALE || scale > DECIMAL_MAX_SCALE) { + throw new IllegalArgumentException(String.format( + "Decimal value out of bounds: precision %d, scale %d " + + "(at most %d significant digits and a scale of at most " + + "%d in either direction)", precision, decimal.scale(), + DECIMAL_MAX_PRECISION, DECIMAL_MAX_SCALE)); + } + return decimal; + } + public Date valueToDate(V value) { if (!this.isDate()) { return null; diff --git a/hugegraph-struct/src/test/java/org/apache/hugegraph/query/ConditionTest.java b/hugegraph-struct/src/test/java/org/apache/hugegraph/query/ConditionTest.java index b34fa4e735..8aa3dbd31d 100644 --- a/hugegraph-struct/src/test/java/org/apache/hugegraph/query/ConditionTest.java +++ b/hugegraph-struct/src/test/java/org/apache/hugegraph/query/ConditionTest.java @@ -17,12 +17,31 @@ package org.apache.hugegraph.query; +import java.math.BigDecimal; +import java.util.Arrays; + import org.apache.hugegraph.type.define.HugeKeys; import org.junit.Assert; import org.junit.Test; public class ConditionTest { + @Test + public void testConditionInMatchesNumbersByValue() { + Condition c = new Condition.SyspropRelation(HugeKeys.ID, Condition.RelationType.IN, + Arrays.asList(new BigDecimal("1.00"), + new BigDecimal("2.5"))); + Assert.assertTrue(c.test(new BigDecimal("1.0"))); + Assert.assertTrue(c.test(1)); + Assert.assertTrue(c.test(2.5d)); + Assert.assertFalse(c.test(new BigDecimal("1.01"))); + Assert.assertFalse(c.test("1.0")); + Condition n = new Condition.SyspropRelation(HugeKeys.ID, Condition.RelationType.NOT_IN, + Arrays.asList(new BigDecimal("1.00"))); + Assert.assertFalse(n.test(new BigDecimal("1.0"))); + Assert.assertTrue(n.test(new BigDecimal("1.1"))); + } + @Test public void testConditionBooleanRange() { Condition lt = Condition.lt(HugeKeys.ID, true); diff --git a/hugegraph-struct/src/test/java/org/apache/hugegraph/struct/schema/PropertyKeyTest.java b/hugegraph-struct/src/test/java/org/apache/hugegraph/struct/schema/PropertyKeyTest.java index d8441144f2..4c5f1d2cc8 100644 --- a/hugegraph-struct/src/test/java/org/apache/hugegraph/struct/schema/PropertyKeyTest.java +++ b/hugegraph-struct/src/test/java/org/apache/hugegraph/struct/schema/PropertyKeyTest.java @@ -17,11 +17,15 @@ package org.apache.hugegraph.struct.schema; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.Arrays; +import java.util.List; import java.util.Date; import java.util.Set; import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.serializer.BytesBuffer; import org.apache.hugegraph.type.define.Cardinality; import org.apache.hugegraph.type.define.DataType; import org.apache.hugegraph.util.DateUtil; @@ -67,4 +71,114 @@ public void testSetDefaultValueCollapsesDuplicatesAndReturnsSet() { Assert.assertEquals(1, values.size()); Assert.assertTrue(values.contains(DateUtil.parse(formatted))); } + + @Test + public void testDecimalPropertyRoundTripAndSchema() { + PropertyKey propertyKey = new PropertyKey(null, IdGenerator.of(2), + "balance"); + propertyKey.dataType(DataType.DECIMAL); + Assert.assertTrue(propertyKey.dataType().isDecimal()); + Assert.assertFalse(propertyKey.dataType().isNumber()); + Assert.assertTrue(propertyKey.convert2Groovy(false).contains(".asDecimal()")); + + // uint256 max survives the struct BytesBuffer used by the store + BigDecimal value = new BigDecimal( + "115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"); + BytesBuffer buffer = BytesBuffer.allocate(64); + buffer.writeProperty(DataType.DECIMAL, value); + Object read = BytesBuffer.wrap(buffer.bytes()) + .readProperty(DataType.DECIMAL); + Assert.assertEquals(value, read); + + BigDecimal wei = new BigDecimal("1.000000000000000001"); + buffer = BytesBuffer.allocate(64); + buffer.writeProperty(DataType.DECIMAL, wei); + read = BytesBuffer.wrap(buffer.bytes()).readProperty(DataType.DECIMAL); + Assert.assertEquals(wei, read); + Assert.assertEquals(18, ((BigDecimal) read).scale()); + } + + @Test + public void testDecimalValueConversion() { + // The struct copy must convert the same inputs as the server copy: + // strings (userdata and JSON), integral numbers, BigInteger + PropertyKey propertyKey = new PropertyKey(null, IdGenerator.of(3), + "balance"); + propertyKey.dataType(DataType.DECIMAL); + + Assert.assertEquals(new BigDecimal("1.5"), + propertyKey.validValueOrThrow("1.5")); + Assert.assertEquals(new BigDecimal("42"), + propertyKey.validValueOrThrow(42L)); + Assert.assertEquals(new BigDecimal("7"), + propertyKey.validValueOrThrow(7)); + String uint256Max = "115792089237316195423570985008687907853" + + "269984665640564039457584007913129639935"; + Assert.assertEquals(new BigDecimal(uint256Max), + propertyKey.validValueOrThrow(uint256Max)); + Assert.assertEquals(new BigDecimal(uint256Max), + propertyKey.validValueOrThrow( + new BigInteger(uint256Max))); + Assert.assertEquals(new BigDecimal("0.000000000000000001"), + propertyKey.validValueOrThrow("1E-18")); + // already the expected type: returned as is + BigDecimal exact = new BigDecimal("-1.50"); + Assert.assertSame(exact, propertyKey.validValueOrThrow(exact)); + + Assert.assertThrows(IllegalArgumentException.class, () -> { + propertyKey.validValueOrThrow("1,5"); + }); + Assert.assertThrows(IllegalArgumentException.class, () -> { + propertyKey.validValueOrThrow(new Date()); + }); + // bounds: a huge exponent must not reach the store + for (String bad : new String[]{"1E+999999999", "1E-999999999"}) { + Assert.assertThrows(IllegalArgumentException.class, () -> { + propertyKey.validValueOrThrow(bad); + }); + } + for (int scale : new int[]{Integer.MIN_VALUE, Integer.MAX_VALUE}) { + Assert.assertThrows(IllegalArgumentException.class, () -> { + propertyKey.validValueOrThrow( + new BigDecimal(java.math.BigInteger.ONE, scale)); + }); + } + Assert.assertThrows(IllegalArgumentException.class, () -> { + propertyKey.validValueOrThrow(new BigDecimal("1E+999999999")); + }); + Assert.assertEquals(new BigDecimal("1E+128"), + propertyKey.validValueOrThrow("1E+128")); + } + + @Test + public void testDefaultValueNormalizedToDecimal() { + // Userdata reloaded from JSON keeps ~default_value as a String; + // defaultValue() must hand back a BigDecimal, exactly + PropertyKey propertyKey = new PropertyKey(null, IdGenerator.of(4), + "balance"); + propertyKey.dataType(DataType.DECIMAL); + propertyKey.userdata(Userdata.DEFAULT_VALUE, "1000000000000000000001"); + + Object value = propertyKey.defaultValue(); + Assert.assertTrue("DEFAULT_VALUE should be a BigDecimal, was " + + (value == null ? "null" : value.getClass()), + value instanceof BigDecimal); + Assert.assertEquals(new BigDecimal("1000000000000000000001"), value); + + // a number literal in the JSON is exact as long as it is integral + propertyKey.userdata(Userdata.DEFAULT_VALUE, 5L); + Assert.assertEquals(new BigDecimal("5"), propertyKey.defaultValue()); + + // list cardinality: every element converted + PropertyKey listKey = new PropertyKey(null, IdGenerator.of(5), + "limits"); + listKey.dataType(DataType.DECIMAL); + listKey.cardinality(Cardinality.LIST); + listKey.userdata(Userdata.DEFAULT_VALUE, Arrays.asList("1", "2.5")); + Object list = listKey.defaultValue(); + Assert.assertTrue(list instanceof List); + Assert.assertEquals(Arrays.asList(new BigDecimal("1"), + new BigDecimal("2.5")), list); + } }