Skip to content

Kill the naked int/long size: ZstdByteSize value type #98

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dfa1 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Kill the naked int/long size: ZstdByteSize value type #98

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 28 additions & 27 deletions .github/smoke/src/test/java/SmokeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// workflow can pin the released version and the matrix's native jar.

import io.github.dfa1.zstd.Zstd;
import io.github.dfa1.zstd.ZstdByteSize;
import io.github.dfa1.zstd.ZstdCompressContext;
import io.github.dfa1.zstd.ZstdCompressDictionary;
import io.github.dfa1.zstd.ZstdCompressionLevel;
Expand Down Expand Up @@ -67,7 +68,7 @@ class SmokeTest {
@BeforeAll
static void trainDictionary() {
List<byte[]> samples = jsonSamples();
dict = ZstdDictionary.train(samples, 8 * 1024);
dict = ZstdDictionary.train(samples, ZstdByteSize.ofKiB(8));
message = samples.get(7);
}

Expand All @@ -82,13 +83,13 @@ void versionAndSizing() {
check(min < max, "minCompressionLevel() >= maxCompressionLevel()");
check(def >= min && def <= max, "defaultCompressionLevel() out of [min,max]");

check(Zstd.compressBound(1000) >= 1000, "compressBound() below input size");
check(Zstd.estimateCompressContextSize(ZstdCompressionLevel.DEFAULT) > 0,
check(Zstd.compressBound(new ZstdByteSize(1000)).value() >= 1000, "compressBound() below input size");
check(Zstd.estimateCompressContextSize(ZstdCompressionLevel.DEFAULT).value() > 0,
"estimateCompressContextSize() not positive");
check(Zstd.estimateDecompressContextSize() > 0, "estimateDecompressContextSize() not positive");
check(Zstd.estimateCompressDictSize(4096, ZstdCompressionLevel.DEFAULT) > 0,
check(Zstd.estimateDecompressContextSize().value() > 0, "estimateDecompressContextSize() not positive");
check(Zstd.estimateCompressDictSize(new ZstdByteSize(4096), ZstdCompressionLevel.DEFAULT).value() > 0,
"estimateCompressDictSize() not positive");
check(Zstd.estimateDecompressDictSize(4096) > 0, "estimateDecompressDictSize() not positive");
check(Zstd.estimateDecompressDictSize(new ZstdByteSize(4096)).value() > 0, "estimateDecompressDictSize() not positive");
}

@Test
Expand All @@ -108,11 +109,11 @@ void explicitBoundDecompress() {
byte[] original = sampleText();
byte[] compressed = Zstd.compress(original);

byte[] restored = Zstd.decompress(compressed, original.length);
byte[] restored = Zstd.decompress(compressed, new ZstdByteSize(original.length));
checkArrayEquals(original, restored, "decompress(byte[], maxSize) mismatch");

try {
Zstd.decompress(compressed, 1);
Zstd.decompress(compressed, new ZstdByteSize(1));
throw new AssertionError("expected ZstdException for an undersized maxSize on " + PLATFORM);
} catch (ZstdException expected) {
check(expected.code() == ZstdErrorCode.DST_SIZE_TOO_SMALL,
Expand All @@ -126,7 +127,7 @@ void zeroCopyFrameSize() {
byte[] compressed = Zstd.compress(original);
try (Arena arena = Arena.ofConfined()) {
MemorySegment frame = toNative(arena, compressed);
check(Zstd.decompressedSize(frame) == original.length, "decompressedSize(MemorySegment) mismatch");
check(Zstd.decompressedSize(frame).value() == original.length, "decompressedSize(MemorySegment) mismatch");
}
}

Expand All @@ -142,7 +143,7 @@ void corruptAndInvalidInput() {
// payload happens to compress on a given platform.
corrupted[corrupted.length - 1] ^= 0xFF;
try {
Zstd.decompress(corrupted, original.length);
Zstd.decompress(corrupted, new ZstdByteSize(original.length));
throw new AssertionError("expected ZstdException for a corrupted checksum on " + PLATFORM);
} catch (ZstdException expected) {
check(expected.code() == ZstdErrorCode.CHECKSUM_WRONG,
Expand All @@ -168,8 +169,8 @@ void frameIntrospection() {
check(!ZstdFrame.isZstdFrame("plain text, not zstd".getBytes(StandardCharsets.UTF_8)),
"isZstdFrame(byte[]) true for non-zstd data");
check(ZstdFrame.compressedSize(compressed) == compressed.length, "compressedSize(byte[]) mismatch");
check(ZstdFrame.decompressedSize(compressed) == original.length, "decompressedSize(byte[]) mismatch");
check(ZstdFrame.decompressedBound(compressed) >= original.length, "decompressedBound(byte[]) too small");
check(ZstdFrame.decompressedSize(compressed).value() == original.length, "decompressedSize(byte[]) mismatch");
check(ZstdFrame.decompressedBound(compressed).value() >= original.length, "decompressedBound(byte[]) too small");
check(ZstdFrame.decompressionMargin(compressed) >= 0, "decompressionMargin(byte[]) negative");
check(!ZstdFrame.dictId(compressed).isPresent(), "dictId(byte[]) expected NONE for a non-dictionary frame");
check(!ZstdFrame.isSkippableFrame(compressed), "isSkippableFrame(byte[]) true for a standard frame");
Expand All @@ -178,7 +179,7 @@ void frameIntrospection() {
check(headerSize > 0 && headerSize <= compressed.length, "headerSize(byte[]) out of range");

ZstdFrameHeader header = ZstdFrame.header(compressed);
check(header.contentSize().isPresent() && header.contentSize().getAsLong() == original.length,
check(header.contentSize().isPresent() && header.contentSize().get().value() == original.length,
"header(byte[]).contentSize() mismatch");
check(header.frameType() == ZstdFrameType.STANDARD, "header(byte[]).frameType() expected STANDARD");
check(header.headerSize() == headerSize, "header(byte[]).headerSize() disagreed with headerSize(byte[])");
Expand Down Expand Up @@ -209,13 +210,13 @@ void compressContextAdvancedParameters() {
.parameter(ZstdCompressParameter.STRATEGY, 3);

byte[] compressed = cctx.compress(original);
checkArrayEquals(original, Zstd.decompress(compressed, original.length),
checkArrayEquals(original, Zstd.decompress(compressed, new ZstdByteSize(original.length)),
"advanced-parameter round-trip mismatch");
check(cctx.sizeOf() > 0, "cctx.sizeOf() not positive");
check(cctx.sizeOf().value() > 0, "cctx.sizeOf() not positive");

cctx.reset(ZstdResetDirective.SESSION_AND_PARAMETERS);
byte[] afterReset = cctx.level(new ZstdCompressionLevel(3)).compress(original);
checkArrayEquals(original, Zstd.decompress(afterReset, original.length),
checkArrayEquals(original, Zstd.decompress(afterReset, new ZstdByteSize(original.length)),
"compress after SESSION_AND_PARAMETERS reset mismatch");
}
}
Expand All @@ -228,7 +229,7 @@ void decompressContextAdvanced() {
dctx.windowLogMax(24);
byte[] restored = dctx.decompress(cctx.compress(original), original.length);
checkArrayEquals(original, restored, "windowLogMax() decompress round-trip mismatch");
check(dctx.sizeOf() > 0, "dctx.sizeOf() not positive");
check(dctx.sizeOf().value() > 0, "dctx.sizeOf() not positive");

dctx.reset(ZstdResetDirective.PARAMETERS);
dctx.parameter(ZstdDecompressParameter.WINDOW_LOG_MAX, 24);
Expand Down Expand Up @@ -269,7 +270,7 @@ void multiThreadRoundTrip() {
try (ZstdCompressContext cctx = new ZstdCompressContext()) {
cctx.parameter(ZstdCompressParameter.NB_WORKERS, 2);
byte[] compressed = cctx.compress(original);
checkArrayEquals(original, Zstd.decompress(compressed, original.length),
checkArrayEquals(original, Zstd.decompress(compressed, new ZstdByteSize(original.length)),
"multithreaded (NB_WORKERS=2) round-trip mismatch");
}
}
Expand All @@ -293,8 +294,8 @@ void digestedDictionaries() {
byte[] compressed = cctx.compress(message, cdict);
byte[] restored = dctx.decompress(compressed, message.length, ddict);
checkArrayEquals(message, restored, "digested-dictionary round-trip mismatch");
check(cdict.sizeOf() > 0, "ZstdCompressDictionary.sizeOf() not positive");
check(ddict.sizeOf() > 0, "ZstdDecompressDictionary.sizeOf() not positive");
check(cdict.sizeOf().value() > 0, "ZstdCompressDictionary.sizeOf() not positive");
check(ddict.sizeOf().value() > 0, "ZstdDecompressDictionary.sizeOf() not positive");
}
}

Expand All @@ -306,7 +307,7 @@ void zeroCopyContextCompression() {
byte[] original = sampleText();
MemorySegment src = toNative(arena, original);

MemorySegment dst = arena.allocate(Zstd.compressBound(src.byteSize()));
MemorySegment dst = arena.allocate(Zstd.compressBound(new ZstdByteSize(src.byteSize())).value());
long written = cctx.compress(dst, src);
MemorySegment restored = arena.allocate(original.length);
long read = dctx.decompress(restored, dst.asSlice(0, written));
Expand All @@ -318,18 +319,18 @@ void zeroCopyContextCompression() {
@Test
void streamingZeroCopy() {
try (ZstdCompressStream cs = new ZstdCompressStream()) {
check(cs.sizeOf() > 0, "no-arg ZstdCompressStream.sizeOf() not positive");
check(cs.sizeOf().value() > 0, "no-arg ZstdCompressStream.sizeOf() not positive");
}

byte[] original = sampleText();
try (Arena arena = Arena.ofConfined();
ZstdCompressStream cs = new ZstdCompressStream(Zstd.defaultCompressionLevel())) {
MemorySegment src = toNative(arena, original);
MemorySegment dst = arena.allocate(Zstd.compressBound(src.byteSize()));
MemorySegment dst = arena.allocate(Zstd.compressBound(new ZstdByteSize(src.byteSize())).value());
ZstdStreamResult result = cs.compress(dst, src, ZstdEndDirective.END);
check(result.isComplete(), "single-shot ZstdCompressStream.compress(END) did not complete");
check(result.bytesConsumed() == original.length, "ZstdCompressStream consumed byte count mismatch");
check(cs.sizeOf() > 0, "ZstdCompressStream.sizeOf() not positive");
check(cs.sizeOf().value() > 0, "ZstdCompressStream.sizeOf() not positive");

// Reads a native struct (ZSTD_frameProgression) through fixed field
// offsets — worth checking per platform since padding/alignment is
Expand All @@ -346,7 +347,7 @@ void streamingZeroCopy() {
check(decodeResult.isComplete(), "ZstdDecompressStream.decompress() did not complete");
checkArrayEquals(original, toByteArray(out, decodeResult.bytesProduced()),
"zero-copy streaming round-trip mismatch");
check(ds.sizeOf() > 0, "ZstdDecompressStream.sizeOf() not positive");
check(ds.sizeOf().value() > 0, "ZstdDecompressStream.sizeOf() not positive");
}
}
}
Expand All @@ -365,11 +366,11 @@ void streamingIo() throws IOException {

ByteArrayOutputStream sinkPledged = new ByteArrayOutputStream();
try (ZstdOutputStream zout =
ZstdOutputStream.withPledgedSize(sinkPledged, new ZstdCompressionLevel(5), original.length)) {
ZstdOutputStream.withPledgedSize(sinkPledged, new ZstdCompressionLevel(5), new ZstdByteSize(original.length))) {
zout.write(original);
}
byte[] pledgedFrame = sinkPledged.toByteArray();
check(ZstdFrame.header(pledgedFrame).contentSize().orElseThrow() == original.length,
check(ZstdFrame.header(pledgedFrame).contentSize().orElseThrow().value() == original.length,
"withPledgedSize() did not stamp the declared content size into the frame header");
checkArrayEquals(original, Zstd.decompress(pledgedFrame), "withPledgedSize() round-trip mismatch");
}
Expand Down
27 changes: 27 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,33 @@ git tags, which trigger publication to Maven Central.
## [Unreleased]

### Changed
- **Breaking:** every public API that took or returned a naked `int`/`long` byte
size or count now takes/returns a `ZstdByteSize` value type, which rejects
negative values at construction (throwing `IllegalArgumentException`) — once
you hold a `ZstdByteSize` it is guaranteed valid, no more wondering whether a
raw `long` from the library still needs checking. Sizes that must fit a
`byte[]` narrow via `ZstdByteSize.toIntExact()` (throwing
`ArithmeticException` above `Integer.MAX_VALUE`), matching the JDK's own
`Math.toIntExact`; native and streaming totals pass through as `long`. A
frame-declared content size that is a zstd sentinel, or otherwise invalid
(including the unsigned range above `Long.MAX_VALUE`, read as a negative
`long`), fails fast with `ZstdException` via
`ZstdByteSize.fromFrameContentSize(long)` rather than reaching the
constructor's generic negative-value guard; `ZstdFrameHeader.contentSize()`
is now `Optional<ZstdByteSize>` (was `OptionalLong`) via the sibling
`ZstdByteSize.fromFrameHeaderContentSize(long)`. Affects
`Zstd.decompress(byte[], …)`, `Zstd.compressBound`,
`Zstd.estimateCompressContextSize`/`estimateDecompressContextSize`/
`estimateCompressDictSize`/`estimateDecompressDictSize`,
`Zstd.decompressedSize(MemorySegment)`,
`ZstdDictionary.train`/`trainCover`/`trainFastCover`/`finalizeFrom`,
`ZstdOutputStream.withPledgedSize`, `ZstdFrame.decompressedSize`/
`decompressedBound`, `ZstdFrameHeader.contentSize`, and `sizeOf()` on
`ZstdCompressContext`/`ZstdDecompressContext`/`ZstdCompressStream`/
`ZstdDecompressStream`/`ZstdCompressDictionary`/`ZstdDecompressDictionary`.
Use `new ZstdByteSize(n)`, or `ZstdByteSize.ofKiB(n)`/`ofMiB(n)` for a size
expressed in KiB/MiB.
([#96](https://github.com/dfa1/zstd-java/issues/96))
- **Breaking:** every public API that took a raw `int` compression level now
takes a `ZstdCompressionLevel` value type, which validates the level against
the linked libzstd's accepted range at construction (throwing
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ import io.github.dfa1.zstd.*;
import java.util.List;

List<byte[]> samples = ...; // representative records
ZstdDictionary dict = ZstdDictionary.train(samples, 8 * 1024);
ZstdDictionary dict = ZstdDictionary.train(samples, ZstdByteSize.ofKiB(8));

byte[] message = ...;
try (ZstdCompressContext cctx = new ZstdCompressContext();
Expand Down
5 changes: 4 additions & 1 deletion benchmark/src/main/java/io/github/dfa1/zstd/bench/CompressBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import io.airlift.compress.v3.zstd.ZstdJavaCompressor;
import io.github.dfa1.zstd.Zstd;
import io.github.dfa1.zstd.ZstdByteSize;
import io.github.dfa1.zstd.ZstdCompressContext;
import io.github.dfa1.zstd.ZstdCompressionLevel;
import java.lang.foreign.Arena;
Expand Down Expand Up @@ -36,6 +37,8 @@
@Measurement(iterations = 5)
public class CompressBenchmark {

// 1 KiB, 64 KiB, 1 MiB, 64 MiB — JMH @Param requires compile-time constant
// Strings, so these can't be expressed via ZstdByteSize.ofKiB/ofMiB.
@Param({"1024", "65536", "1048576", "67108864"})
private int size;

Expand All @@ -57,7 +60,7 @@ public class CompressBenchmark {
@Setup(Level.Trial)
public void setup() {
src = BenchData.generate(size);
int bound = (int) Zstd.compressBound(size);
int bound = Zstd.compressBound(new ZstdByteSize(size)).toIntExact();

ffmCtx = new ZstdCompressContext().level(new ZstdCompressionLevel(level));
ffmDst = new byte[bound];
Expand Down
2 changes: 2 additions & 0 deletions benchmark/src/main/java/io/github/dfa1/zstd/bench/DecompressBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
@Measurement(iterations = 5)
public class DecompressBenchmark {

// 1 KiB, 64 KiB, 1 MiB, 64 MiB — JMH @Param requires compile-time constant
// Strings, so these can't be expressed via ZstdByteSize.ofKiB/ofMiB.
@Param({"1024", "65536", "1048576", "67108864"})
private int size;

Expand Down
3 changes: 2 additions & 1 deletion benchmark/src/main/java/io/github/dfa1/zstd/bench/GoldenCorpusBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import static java.lang.foreign.ValueLayout.JAVA_BYTE;

import io.github.dfa1.zstd.Zstd;
import io.github.dfa1.zstd.ZstdByteSize;
import io.github.dfa1.zstd.ZstdCompressContext;
import io.github.dfa1.zstd.ZstdCompressionLevel;
import io.github.dfa1.zstd.ZstdDecompressContext;
Expand Down Expand Up @@ -91,7 +92,7 @@ public void setup() {

cctx = new ZstdCompressContext().level(new ZstdCompressionLevel(level));
dctx = new ZstdDecompressContext();
bound = (int) Zstd.compressBound(srcSize);
bound = Zstd.compressBound(new ZstdByteSize(srcSize)).toIntExact();
compressDst = new byte[bound];

arena = Arena.ofConfined();
Expand Down
5 changes: 4 additions & 1 deletion benchmark/src/main/java/io/github/dfa1/zstd/bench/MultiThreadCompressBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import static java.lang.foreign.ValueLayout.JAVA_BYTE;

import io.github.dfa1.zstd.Zstd;
import io.github.dfa1.zstd.ZstdByteSize;
import io.github.dfa1.zstd.ZstdCompressContext;
import io.github.dfa1.zstd.ZstdCompressionLevel;
import io.github.dfa1.zstd.ZstdCompressParameter;
Expand Down Expand Up @@ -40,6 +41,8 @@
@Measurement(iterations = 5)
public class MultiThreadCompressBenchmark {

// 1 MiB, 64 MiB — JMH @Param requires compile-time constant Strings, so
// these can't be expressed via ZstdByteSize.ofMiB.
@Param({"1048576", "67108864"})
private int size;

Expand All @@ -58,7 +61,7 @@ public class MultiThreadCompressBenchmark {
@Setup(Level.Trial)
public void setup() {
byte[] src = BenchData.generate(size);
int bound = (int) Zstd.compressBound(size);
int bound = Zstd.compressBound(new ZstdByteSize(size)).toIntExact();

ctx = new ZstdCompressContext().level(new ZstdCompressionLevel(level));
if (nbWorkers > 0) {
Expand Down
10 changes: 5 additions & 5 deletions docs/how-to.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ dictionary compresses each one far smaller than it could be alone. Train one on
representative samples:

```java
ZstdDictionary dict = ZstdDictionary.train(sampleRecords, 16 * 1024);
ZstdDictionary dict = ZstdDictionary.train(sampleRecords, ZstdByteSize.ofKiB(16));

try (ZstdCompressContext cctx = new ZstdCompressContext();
ZstdDecompressContext dctx = new ZstdDecompressContext()) {
Expand Down Expand Up @@ -124,8 +124,8 @@ hands zstd the segment address directly: no copy in, no copy out, no GC churn.
try (Arena arena = Arena.ofConfined();
ZstdDecompressContext dctx = new ZstdDecompressContext()) {
MemorySegment frame = reader.mmapSlice(); // already native
long n = Zstd.decompressedSize(frame); // read header, no copy
MemorySegment out = arena.allocate(n); // becomes the backing buffer
ZstdByteSize n = Zstd.decompressedSize(frame); // read header, no copy
MemorySegment out = arena.allocate(n.value()); // becomes the backing buffer
dctx.decompress(out, frame); // native → native
}
```
Expand All @@ -144,7 +144,7 @@ The segment-API map:
| decompress + dict | `ZstdDecompressContext.decompress(byte[], int, ZstdDecompressDictionary)` | `ZstdDecompressContext.decompress(dst, src, ZstdDecompressDictionary)` |
| size output (no copy) | frame header via `Zstd.decompress(byte[])` | `Zstd.decompressedSize(MemorySegment)` |

Size `dst` with `Zstd.compressBound(srcSize)` for compression, or
Size `dst` with `Zstd.compressBound(new ZstdByteSize(srcSize))` for compression, or
`Zstd.decompressedSize(frame)` for decompression.

## Let the codec size and allocate the output
Expand Down Expand Up @@ -231,7 +231,7 @@ can't size the arena (see [the explanation](zero-copy.md)). Tell the encoder the
total up front and it stamps the content size into the header:

```java
try (var zout = ZstdOutputStream.withPledgedSize(sink, new ZstdCompressionLevel(6), data.length)) {
try (var zout = ZstdOutputStream.withPledgedSize(sink, new ZstdCompressionLevel(6), new ZstdByteSize(data.length))) {
zout.write(data); // pledge must equal bytes written
}
MemorySegment src = MemorySegment.ofBuffer(mmap); // downstream, in a mapped reader
Expand Down
4 changes: 2 additions & 2 deletions docs/tutorial.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,8 @@ the `MemorySegment` overloads hand zstd the segment address directly, so there i
try (Arena arena = Arena.ofConfined();
ZstdDecompressContext dctx = new ZstdDecompressContext()) {
MemorySegment frame = reader.mmapSlice(); // already native — never touches the heap
long n = Zstd.decompressedSize(frame); // read the header, no copy
MemorySegment out = arena.allocate(n); // this segment *is* the output buffer
ZstdByteSize n = Zstd.decompressedSize(frame); // read the header, no copy
MemorySegment out = arena.allocate(n.value()); // this segment *is* the output buffer
dctx.decompress(out, frame); // native → native
}
```
Expand Down
Loading
Loading