From fecfedd1a4bce3846a312fb4a3fb6264d1d542a5 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 31 Mar 2022 09:33:11 +0100 Subject: [PATCH] feat: compute buffer size ahead of time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current implementation loops over message fields and each field encoder creates a `Uint8Array` which is added to a `Uint8ArrayList` and stitched together at the end of the process. Other implementations compute the expected length of the serialized message, allocate a buffer that size then have encoders write their data into it at the correct offsets. This PR makes protons compute the final message buffer size first in the same way. Thing is, it doesn't seem to make a huge amount of difference to performance. Before: ``` Running "Encode/Decode" suite... Progress: 100% pbjs: 12 166 ops/s, ±3.92% | 5.12% slower protons: 9 755 ops/s, ±2.19% | slowest, 23.93% slower protobufjs: 12 823 ops/s, ±2.02% | fastest Finished 3 cases! Fastest: protobufjs Slowest: protons ``` After: ``` Running "Encode/Decode" suite... Progress: 100% pbjs: 11 866 ops/s, ±3.43% | 2.05% slower protons: 9 356 ops/s, ±2.45% | slowest, 22.77% slower protobufjs: 12 114 ops/s, ±2.16% | fastest Finished 3 cases! Fastest: protobufjs Slowest: protons ``` --- packages/protons-runtime/src/codecs/bool.ts | 6 +- packages/protons-runtime/src/codecs/bytes.ts | 10 +-- packages/protons-runtime/src/codecs/codec.ts | 2 +- packages/protons-runtime/src/codecs/double.ts | 8 +- packages/protons-runtime/src/codecs/enum.ts | 7 +- .../protons-runtime/src/codecs/fixed32.ts | 8 +- .../protons-runtime/src/codecs/fixed64.ts | 8 +- packages/protons-runtime/src/codecs/float.ts | 8 +- packages/protons-runtime/src/codecs/int32.ts | 7 +- packages/protons-runtime/src/codecs/int64.ts | 7 +- .../protons-runtime/src/codecs/message.ts | 85 ++++++++++++------- .../protons-runtime/src/codecs/sfixed32.ts | 8 +- .../protons-runtime/src/codecs/sfixed64.ts | 8 +- packages/protons-runtime/src/codecs/sint32.ts | 8 +- packages/protons-runtime/src/codecs/sint64.ts | 7 +- packages/protons-runtime/src/codecs/string.ts | 12 ++- packages/protons-runtime/src/codecs/uint32.ts | 12 +-- packages/protons-runtime/src/codecs/uint64.ts | 8 +- packages/protons-runtime/src/decode.ts | 4 +- packages/protons-runtime/src/encode.ts | 11 ++- .../protons-runtime/src/utils/big-varint.ts | 19 ++--- .../protons-runtime/src/utils/long-bits.ts | 4 +- packages/protons-runtime/src/utils/varint.ts | 22 ++--- 23 files changed, 132 insertions(+), 147 deletions(-) diff --git a/packages/protons-runtime/src/codecs/bool.ts b/packages/protons-runtime/src/codecs/bool.ts index aa3c0d1e..ab45ca06 100644 --- a/packages/protons-runtime/src/codecs/bool.ts +++ b/packages/protons-runtime/src/codecs/bool.ts @@ -4,8 +4,10 @@ const encodingLength: EncodingLengthFunction = function boolEncodingLen return 1 } -const encode: EncodeFunction = function boolEncode (value) { - return Uint8Array.from([value ? 1 : 0]) +const encode: EncodeFunction = function boolEncode (val, buf, offset) { + buf.set(offset, val ? 1 : 0) + + return offset + encodingLength(val) } const decode: DecodeFunction = function boolDecode (buffer, offset) { diff --git a/packages/protons-runtime/src/codecs/bytes.ts b/packages/protons-runtime/src/codecs/bytes.ts index 37123f22..cd36bf65 100644 --- a/packages/protons-runtime/src/codecs/bytes.ts +++ b/packages/protons-runtime/src/codecs/bytes.ts @@ -1,5 +1,4 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { unsigned } from '../utils/varint.js' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' @@ -8,12 +7,11 @@ const encodingLength: EncodingLengthFunction = function bytesEncodin return unsigned.encodingLength(len) + len } -const encode: EncodeFunction = function bytesEncode (val) { - const prefix = new Uint8Array(unsigned.encodingLength(val.byteLength)) +const encode: EncodeFunction = function bytesEncode (val, buf, offset) { + offset = unsigned.encode(val.byteLength, buf, offset) + buf.write(val, offset) - unsigned.encode(val.byteLength, prefix) - - return new Uint8ArrayList(prefix, val) + return offset + val.byteLength } const decode: DecodeFunction = function bytesDecode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/codec.ts b/packages/protons-runtime/src/codecs/codec.ts index 7332fd73..52c9d1c9 100644 --- a/packages/protons-runtime/src/codecs/codec.ts +++ b/packages/protons-runtime/src/codecs/codec.ts @@ -11,7 +11,7 @@ export enum CODEC_TYPES { } export interface EncodeFunction { - (value: T): Uint8Array | Uint8ArrayList + (value: T, buf: Uint8ArrayList, offset: number): number } export interface DecodeFunction { diff --git a/packages/protons-runtime/src/codecs/double.ts b/packages/protons-runtime/src/codecs/double.ts index 1d887fcb..a93c772b 100644 --- a/packages/protons-runtime/src/codecs/double.ts +++ b/packages/protons-runtime/src/codecs/double.ts @@ -1,15 +1,13 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' const encodingLength: EncodingLengthFunction = function doubleEncodingLength () { return 8 } -const encode: EncodeFunction = function doubleEncode (val) { - const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val))) - buf.setFloat64(0, val, true) +const encode: EncodeFunction = function doubleEncode (val, buf, offset) { + buf.setFloat64(offset, val, true) - return buf + return offset + encodingLength(val) } const decode: DecodeFunction = function doubleDecode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/enum.ts b/packages/protons-runtime/src/codecs/enum.ts index 26f85789..fb2d3ca2 100644 --- a/packages/protons-runtime/src/codecs/enum.ts +++ b/packages/protons-runtime/src/codecs/enum.ts @@ -10,14 +10,11 @@ export function enumeration (e: T): Codec { return unsigned.encodingLength(index) } - const encode: EncodeFunction = function enumEncode (val) { + const encode: EncodeFunction = function enumEncode (val, buf, offset) { const keys = Object.keys(e) const index = keys.indexOf(val) - const buf = new Uint8Array(unsigned.encodingLength(index)) - unsigned.encode(index, buf) - - return buf + return unsigned.encode(index, buf, offset) } const decode: DecodeFunction = function enumDecode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/fixed32.ts b/packages/protons-runtime/src/codecs/fixed32.ts index cd29e3d6..50a74785 100644 --- a/packages/protons-runtime/src/codecs/fixed32.ts +++ b/packages/protons-runtime/src/codecs/fixed32.ts @@ -1,15 +1,13 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' const encodingLength: EncodingLengthFunction = function fixed32EncodingLength () { return 4 } -const encode: EncodeFunction = function fixed32Encode (val) { - const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val))) - buf.setInt32(0, val, true) +const encode: EncodeFunction = function fixed32Encode (val, buf, offset) { + buf.setInt32(offset, val, true) - return buf + return offset + encodingLength(val) } const decode: DecodeFunction = function fixed32Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/fixed64.ts b/packages/protons-runtime/src/codecs/fixed64.ts index d8113f1b..84a31481 100644 --- a/packages/protons-runtime/src/codecs/fixed64.ts +++ b/packages/protons-runtime/src/codecs/fixed64.ts @@ -1,15 +1,13 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' const encodingLength: EncodingLengthFunction = function int64EncodingLength (val) { return 8 } -const encode: EncodeFunction = function int64Encode (val) { - const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val))) - buf.setBigInt64(0, val, true) +const encode: EncodeFunction = function int64Encode (val, buf, offset) { + buf.setBigInt64(offset, val, true) - return buf + return offset + encodingLength(val) } const decode: DecodeFunction = function int64Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/float.ts b/packages/protons-runtime/src/codecs/float.ts index 10270af0..18698c1c 100644 --- a/packages/protons-runtime/src/codecs/float.ts +++ b/packages/protons-runtime/src/codecs/float.ts @@ -1,15 +1,13 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' const encodingLength: EncodingLengthFunction = function floatEncodingLength () { return 4 } -const encode: EncodeFunction = function floatEncode (val) { - const buf = new Uint8ArrayList(new Uint8Array(encodingLength(1))) - buf.setFloat32(0, val, true) +const encode: EncodeFunction = function floatEncode (val, buf, offset) { + buf.setFloat32(offset, val, true) - return buf + return offset + encodingLength(val) } const decode: DecodeFunction = function floatDecode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/int32.ts b/packages/protons-runtime/src/codecs/int32.ts index 0889d580..17ec510f 100644 --- a/packages/protons-runtime/src/codecs/int32.ts +++ b/packages/protons-runtime/src/codecs/int32.ts @@ -5,11 +5,8 @@ const encodingLength: EncodingLengthFunction = function int32EncodingLen return signed.encodingLength(val) } -const encode: EncodeFunction = function int32Encode (val) { - const buf = new Uint8Array(encodingLength(val)) - signed.encode(val, buf) - - return buf +const encode: EncodeFunction = function int32Encode (val, buf, offset) { + return signed.encode(val, buf, offset) } const decode: DecodeFunction = function int32Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/int64.ts b/packages/protons-runtime/src/codecs/int64.ts index d58c0a74..161ff973 100644 --- a/packages/protons-runtime/src/codecs/int64.ts +++ b/packages/protons-runtime/src/codecs/int64.ts @@ -5,11 +5,8 @@ const encodingLength: EncodingLengthFunction = function int64EncodingLen return signed.encodingLength(val) } -const encode: EncodeFunction = function int64Encode (val) { - const buf = new Uint8Array(encodingLength(val)) - signed.encode(val, buf) - - return buf +const encode: EncodeFunction = function int64Encode (val, buf, offset) { + return signed.encode(val, buf, offset) } const decode: DecodeFunction = function int64Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/message.ts b/packages/protons-runtime/src/codecs/message.ts index 64cf995a..13de2a04 100644 --- a/packages/protons-runtime/src/codecs/message.ts +++ b/packages/protons-runtime/src/codecs/message.ts @@ -1,7 +1,6 @@ import { unsigned } from '../utils/varint.js' import type { FieldDef, FieldDefs } from '../index.js' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, Codec, CODEC_TYPES } from './codec.js' -import { Uint8ArrayList } from 'uint8arraylist' export interface Factory { new (obj: A): T @@ -9,56 +8,91 @@ export interface Factory { export function message (fieldDefs: FieldDefs): Codec { const encodingLength: EncodingLengthFunction = function messageEncodingLength (val: Record) { + function valueEncodingLength (value: any, fieldNumber: number, fieldDef: FieldDef): number { + if (value == null) { + if (fieldDef.optional === true) { + return 0 + } + + throw new Error(`Non optional field "${fieldDef.name}" was ${value === null ? 'null' : 'undefined'}`) + } + + const key = (fieldNumber << 3) | fieldDef.codec.type + const prefixLength = unsigned.encodingLength(key) + + return fieldDef.codec.encodingLength(value) + prefixLength + } + let length = 0 - for (const fieldDef of Object.values(fieldDefs)) { - length += fieldDef.codec.encodingLength(val[fieldDef.name]) + for (const [fieldNumberStr, fieldDef] of Object.entries(fieldDefs)) { + const fieldNumber = parseInt(fieldNumberStr) + const value = val[fieldDef.name] + + if (value == null && !(fieldDef.optional === true)) { + throw new Error(`Field ${fieldDef.name} cannot be ${value === null ? 'null' : 'undefined'}`) + } + + if (fieldDef.repeats === true) { + if (!Array.isArray(value)) { + throw new Error(`Repeating field ${fieldDef.name} was not an array`) + } + + for (const entry of value) { + length += valueEncodingLength(entry, fieldNumber, fieldDef) + } + + continue + } + + length += valueEncodingLength(value, fieldNumber, fieldDef) } return unsigned.encodingLength(length) + length } - const encode: EncodeFunction> = function messageEncode (val) { - const bytes = new Uint8ArrayList() - - function encodeValue (value: any, fieldNumber: number, fieldDef: FieldDef) { + const encode: EncodeFunction = function messageEncode (val, buf, offset): number { + function encodeValue (value: any, fieldNumber: number, fieldDef: FieldDef, offset: number): number { if (value == null) { if (fieldDef.optional === true) { - return + return offset } throw new Error(`Non optional field "${fieldDef.name}" was ${value === null ? 'null' : 'undefined'}`) } const key = (fieldNumber << 3) | fieldDef.codec.type - const prefix = new Uint8Array(unsigned.encodingLength(key)) - unsigned.encode(key, prefix) - const encoded = fieldDef.codec.encode(value) + offset = unsigned.encode(key, buf, offset) + offset = fieldDef.codec.encode(value, buf, offset) - bytes.append(prefix) - bytes.append(encoded) + return offset } + const length = encodingLength(val) + offset = unsigned.encode(length - unsigned.encodingLength(length), buf, offset) + for (const [fieldNumberStr, fieldDef] of Object.entries(fieldDefs)) { const fieldNumber = parseInt(fieldNumberStr) if (fieldDef.repeats === true) { + // @ts-expect-error cannot use strings to index T if (!Array.isArray(val[fieldDef.name])) { throw new Error(`Repeating field "${fieldDef.name}" was not an array`) } + // @ts-expect-error cannot use strings to index T for (const value of val[fieldDef.name]) { - encodeValue(value, fieldNumber, fieldDef) + offset = encodeValue(value, fieldNumber, fieldDef, offset) } - } else { - encodeValue(val[fieldDef.name], fieldNumber, fieldDef) + + continue } - } - const prefix = new Uint8Array(unsigned.encodingLength(bytes.length)) - unsigned.encode(bytes.length, prefix) + // @ts-expect-error cannot use strings to index T + offset = encodeValue(val[fieldDef.name], fieldNumber, fieldDef, offset) + } - return new Uint8ArrayList(prefix, bytes) + return offset } const decode: DecodeFunction = function messageDecode (buffer, offset) { @@ -78,10 +112,7 @@ export function message (fieldDefs: FieldDefs): Codec { const fieldDef = fieldDefs[fieldNumber] let fieldLength = 0 - // console.info('fieldNumber', fieldNumber, 'wireType', wireType, 'offset', offset) - if (wireType === CODEC_TYPES.VARINT) { - // console.info('decode varint') if (fieldDef != null) { // use the codec if it is available as this could be a bigint const value = fieldDef.codec.decode(buffer, offset) @@ -91,14 +122,11 @@ export function message (fieldDefs: FieldDefs): Codec { fieldLength = unsigned.encodingLength(value) } } else if (wireType === CODEC_TYPES.BIT64) { - // console.info('decode 64bit') fieldLength = 8 } else if (wireType === CODEC_TYPES.LENGTH_DELIMITED) { - // console.info('decode length delimited') const valueLength = unsigned.decode(buffer, offset) fieldLength = valueLength + unsigned.encodingLength(valueLength) } else if (wireType === CODEC_TYPES.BIT32) { - // console.info('decode 32 bit') fieldLength = 4 } else if (wireType === CODEC_TYPES.START_GROUP) { throw new Error('Unsupported wire type START_GROUP') @@ -106,10 +134,7 @@ export function message (fieldDefs: FieldDefs): Codec { throw new Error('Unsupported wire type END_GROUP') } - // console.info('fieldLength', fieldLength) - if (fieldDef != null) { - // console.info('decode', fieldDef.codec.name, fieldDef.name, 'at offset', offset) const value = fieldDef.codec.decode(buffer, offset) if (fieldDef.repeats === true) { @@ -121,8 +146,6 @@ export function message (fieldDefs: FieldDefs): Codec { } else { fields[fieldDef.name] = value } - - // console.info('decoded', value) } offset += fieldLength diff --git a/packages/protons-runtime/src/codecs/sfixed32.ts b/packages/protons-runtime/src/codecs/sfixed32.ts index f165b904..49adbad4 100644 --- a/packages/protons-runtime/src/codecs/sfixed32.ts +++ b/packages/protons-runtime/src/codecs/sfixed32.ts @@ -1,15 +1,13 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' const encodingLength: EncodingLengthFunction = function sfixed32EncodingLength () { return 4 } -const encode: EncodeFunction = function sfixed32Encode (val) { - const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val))) - buf.setInt32(0, val, true) +const encode: EncodeFunction = function sfixed32Encode (val, buf, offset) { + buf.setInt32(offset, val, true) - return buf + return offset + encodingLength(val) } const decode: DecodeFunction = function sfixed32Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/sfixed64.ts b/packages/protons-runtime/src/codecs/sfixed64.ts index 68359b00..440b29f6 100644 --- a/packages/protons-runtime/src/codecs/sfixed64.ts +++ b/packages/protons-runtime/src/codecs/sfixed64.ts @@ -1,15 +1,13 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' const encodingLength: EncodingLengthFunction = function sfixed64EncodingLength () { return 8 } -const encode: EncodeFunction = function sfixed64Encode (val) { - const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val))) - buf.setBigInt64(0, val, true) +const encode: EncodeFunction = function sfixed64Encode (val, buf, offset) { + buf.setBigInt64(offset, val, true) - return buf + return offset + encodingLength(val) } const decode: DecodeFunction = function sfixed64Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/sint32.ts b/packages/protons-runtime/src/codecs/sint32.ts index 3c72c2c0..b5f14c20 100644 --- a/packages/protons-runtime/src/codecs/sint32.ts +++ b/packages/protons-runtime/src/codecs/sint32.ts @@ -5,12 +5,8 @@ const encodingLength: EncodingLengthFunction = function sint32EncodingLe return zigzag.encodingLength(val) } -const encode: EncodeFunction = function svarintEncode (val) { - const buf = new Uint8Array(encodingLength(val)) - - zigzag.encode(val, buf) - - return buf +const encode: EncodeFunction = function svarintEncode (val, buf, offset) { + return zigzag.encode(val, buf, offset) } const decode: DecodeFunction = function svarintDecode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/sint64.ts b/packages/protons-runtime/src/codecs/sint64.ts index 2dc45507..1c0a25c6 100644 --- a/packages/protons-runtime/src/codecs/sint64.ts +++ b/packages/protons-runtime/src/codecs/sint64.ts @@ -5,11 +5,8 @@ const encodingLength: EncodingLengthFunction = function int64EncodingLen return zigzag.encodingLength(val) } -const encode: EncodeFunction = function int64Encode (val) { - const buf = new Uint8Array(encodingLength(val)) - zigzag.encode(val, buf) - - return buf +const encode: EncodeFunction = function int64Encode (val, buf, offset) { + return zigzag.encode(val, buf, offset) } const decode: DecodeFunction = function int64Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/string.ts b/packages/protons-runtime/src/codecs/string.ts index 04216dd9..0416850c 100644 --- a/packages/protons-runtime/src/codecs/string.ts +++ b/packages/protons-runtime/src/codecs/string.ts @@ -2,20 +2,18 @@ import { unsigned } from '../utils/varint.js' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' -import { Uint8ArrayList } from 'uint8arraylist' const encodingLength: EncodingLengthFunction = function stringEncodingLength (val) { const len = uint8ArrayFromString(val).byteLength - return unsigned.encodingLength(len) + len + return unsigned.encodingLength(len) + val.length } -const encode: EncodeFunction = function stringEncode (val) { +const encode: EncodeFunction = function stringEncode (val, buf, offset) { const asBuf = uint8ArrayFromString(val) - const prefix = new Uint8Array(unsigned.encodingLength(asBuf.byteLength)) + offset = unsigned.encode(asBuf.length, buf, offset) + buf.write(asBuf, offset) - unsigned.encode(asBuf.length, prefix) - - return new Uint8ArrayList(prefix, asBuf) + return offset + asBuf.byteLength } const decode: DecodeFunction = function stringDecode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/uint32.ts b/packages/protons-runtime/src/codecs/uint32.ts index 57aa1a79..976f26c3 100644 --- a/packages/protons-runtime/src/codecs/uint32.ts +++ b/packages/protons-runtime/src/codecs/uint32.ts @@ -5,20 +5,12 @@ const encodingLength: EncodingLengthFunction = function uint32EncodingLe return unsigned.encodingLength(val) } -const encode: EncodeFunction = function uint32Encode (val) { - // val = val < 0 ? val + 4294967296 : val - - const buf = new Uint8Array(encodingLength(val)) - - unsigned.encode(val, buf) - - return buf +const encode: EncodeFunction = function uint32Encode (val, buf, offset) { + return unsigned.encode(val, buf, offset) } const decode: DecodeFunction = function uint32Decode (buf, offset) { return unsigned.decode(buf, offset) - - // return value > 2147483647 ? value - 4294967296 : value } export const uint32 = createCodec('uint32', CODEC_TYPES.VARINT, encode, decode, encodingLength) diff --git a/packages/protons-runtime/src/codecs/uint64.ts b/packages/protons-runtime/src/codecs/uint64.ts index 4753114d..d1631052 100644 --- a/packages/protons-runtime/src/codecs/uint64.ts +++ b/packages/protons-runtime/src/codecs/uint64.ts @@ -5,12 +5,8 @@ const encodingLength: EncodingLengthFunction = function uint64EncodingLe return unsigned.encodingLength(val) } -const encode: EncodeFunction = function uint64Encode (val) { - const buf = new Uint8Array(unsigned.encodingLength(val)) - - unsigned.encode(val, buf) - - return buf +const encode: EncodeFunction = function uint64Encode (val, buf, offset) { + return unsigned.encode(val, buf, offset) } const decode: DecodeFunction = function uint64Decode (buf, offset) { diff --git a/packages/protons-runtime/src/decode.ts b/packages/protons-runtime/src/decode.ts index 99b427de..b2436fb7 100644 --- a/packages/protons-runtime/src/decode.ts +++ b/packages/protons-runtime/src/decode.ts @@ -4,8 +4,8 @@ import type { Codec } from './codecs/codec.js' export function decodeMessage (buf: Uint8Array, codec: Codec) { // wrap root message - const prefix = new Uint8Array(unsigned.encodingLength(buf.length)) - unsigned.encode(buf.length, prefix) + const prefix = Buffer.allocUnsafe(unsigned.encodingLength(buf.length)) + unsigned.encode(buf.length, prefix, 0) return codec.decode(new Uint8ArrayList(prefix, buf), 0) } diff --git a/packages/protons-runtime/src/encode.ts b/packages/protons-runtime/src/encode.ts index 83f8bae4..2cc4aa66 100644 --- a/packages/protons-runtime/src/encode.ts +++ b/packages/protons-runtime/src/encode.ts @@ -1,10 +1,13 @@ +import { Uint8ArrayList } from 'uint8arraylist' import type { Codec } from './codecs/codec.js' import { unsigned } from './utils/varint.js' export function encodeMessage (message: T, codec: Codec) { - // unwrap root message - const encoded = codec.encode(message) - const skip = unsigned.encodingLength(unsigned.decode(encoded)) + const len = codec.encodingLength(message) + const buf = new Uint8ArrayList(Buffer.allocUnsafe(len)) - return encoded.slice(skip) + // unwrap root message + codec.encode(message, buf, 0) + const skip = unsigned.encodingLength(unsigned.decode(buf, 0)) + return buf.slice(skip) } diff --git a/packages/protons-runtime/src/utils/big-varint.ts b/packages/protons-runtime/src/utils/big-varint.ts index a899e1e6..d8d54177 100644 --- a/packages/protons-runtime/src/utils/big-varint.ts +++ b/packages/protons-runtime/src/utils/big-varint.ts @@ -14,19 +14,20 @@ export const unsigned = { return i + 1 }, - encode (value: bigint, buf: Uint8ArrayList | Uint8Array) { + encode (value: bigint, buf: Uint8ArrayList | Uint8Array, offset: number): number { const access = accessor(buf) - let offset = 0 while (LIMIT < value) { access.set(offset++, Number(value & LIMIT) | 0x80) value >>= 7n } access.set(offset, Number(value)) + + return offset + 1 }, - decode (buf: Uint8ArrayList | Uint8Array, offset = 0) { + decode (buf: Uint8ArrayList | Uint8Array, offset = 0): bigint { return LongBits.fromBytes(buf, offset).toBigInt(true) } } @@ -40,14 +41,12 @@ export const signed = { return unsigned.encodingLength(value) }, - encode (value: bigint, buf: Uint8ArrayList | Uint8Array, offset = 0) { + encode (value: bigint, buf: Uint8ArrayList | Uint8Array, offset: number): number { if (value < 0n) { - LongBits.fromBigInt(value).toBytes(buf, offset) - - return + return LongBits.fromBigInt(value).write(buf, offset) } - return unsigned.encode(value, buf) + return unsigned.encode(value, buf, offset) }, decode (buf: Uint8ArrayList | Uint8Array, offset = 0) { @@ -60,8 +59,8 @@ export const zigzag = { return unsigned.encodingLength(value >= 0 ? value * 2n : value * -2n - 1n) }, - encode (value: bigint, buf: Uint8ArrayList | Uint8Array, offset = 0) { - LongBits.fromBigInt(value).zzEncode().toBytes(buf, offset) + encode (value: bigint, buf: Uint8ArrayList | Uint8Array, offset: number): number { + return LongBits.fromBigInt(value).zzEncode().write(buf, offset) }, decode (buf: Uint8ArrayList | Uint8Array, offset = 0) { diff --git a/packages/protons-runtime/src/utils/long-bits.ts b/packages/protons-runtime/src/utils/long-bits.ts index ad1ddcdd..3b6f0859 100644 --- a/packages/protons-runtime/src/utils/long-bits.ts +++ b/packages/protons-runtime/src/utils/long-bits.ts @@ -47,7 +47,7 @@ export class LongBits { return new LongBits(hi, lo) } - toBytes (buf: Uint8ArrayList | Uint8Array, offset = 0) { + write (buf: Uint8ArrayList | Uint8Array, offset = 0): number { const access = accessor(buf) while (this.hi > 0) { @@ -62,6 +62,8 @@ export class LongBits { } access.set(offset++, this.lo) + + return offset } static fromBigInt (value: bigint) { diff --git a/packages/protons-runtime/src/utils/varint.ts b/packages/protons-runtime/src/utils/varint.ts index f3b07382..2ca5bf47 100644 --- a/packages/protons-runtime/src/utils/varint.ts +++ b/packages/protons-runtime/src/utils/varint.ts @@ -57,8 +57,7 @@ export const unsigned = { return 10 }, - encode (value: number, buf: Uint8ArrayList | Uint8Array) { - let offset = 0 + encode (value: number, buf: Uint8ArrayList | Uint8Array, offset: number): number { const access = accessor(buf) while (value >= INT) { @@ -72,9 +71,11 @@ export const unsigned = { } access.set(offset, value | 0) + + return offset + 1 }, - decode (buf: Uint8ArrayList | Uint8Array, offset: number = 0) { + decode (buf: Uint8ArrayList | Uint8Array, offset: number) { const access = accessor(buf) let value = 4294967295 // optimizer type-hint, tends to deopt otherwise (?!) @@ -125,9 +126,8 @@ export const signed = { return unsigned.encodingLength(value) }, - encode (value: number, buf: Uint8ArrayList | Uint8Array) { + encode (value: number, buf: Uint8ArrayList | Uint8Array, offset: number): number { if (value < 0) { - let offset = 0 const access = accessor(buf) const bits = LongBits.fromNumber(value) @@ -144,13 +144,13 @@ export const signed = { access.set(offset++, bits.lo) - return + return offset } - unsigned.encode(value, buf) + return unsigned.encode(value, buf, offset) }, - decode (data: Uint8ArrayList | Uint8Array, offset = 0) { + decode (data: Uint8ArrayList | Uint8Array, offset: number) { return unsigned.decode(data, offset) | 0 } } @@ -161,12 +161,12 @@ export const zigzag = { return unsigned.encodingLength(value) }, - encode (value: number, buf: Uint8ArrayList | Uint8Array, offset = 0) { + encode (value: number, buf: Uint8ArrayList | Uint8Array, offset: number): number { value = (value << 1 ^ value >> 31) >>> 0 - return unsigned.encode(value, buf) + return unsigned.encode(value, buf, offset) }, - decode (data: Uint8ArrayList | Uint8Array, offset = 0) { + decode (data: Uint8ArrayList | Uint8Array, offset: number) { const value = unsigned.decode(data, offset) return value >>> 1 ^ -(value & 1) | 0 }