diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index c0bb677..165f5d7 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -12,11 +12,9 @@ jobs: strategy: matrix: - os: [ ubuntu-latest ] - java: [ 11, 17, 21, 25 ] - include: - - os: windows-latest - java: 25 + os: [ ubuntu-latest, windows-latest ] + java: [ 21, 25 ] + fail-fast: false steps: - name: Checkout diff --git a/.github/workflows/release-prepare.yml b/.github/workflows/release-prepare.yml index 2c68c5e..392cbe6 100644 --- a/.github/workflows/release-prepare.yml +++ b/.github/workflows/release-prepare.yml @@ -24,7 +24,7 @@ jobs: - name: Set up JDK uses: actions/setup-java@v4 with: - java-version: 17 + java-version: 21 distribution: 'temurin' cache: 'maven' diff --git a/CLAUDE.md b/CLAUDE.md index 4435e86..9f1a7ff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,13 +5,13 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Build, Test, Run - `mvn clean package` — full build; produces the shaded kernel jar and the kernelspec zip under `jjava-distro/target/`. -- `mvn clean verify` — what CI runs on Linux/macOS. Adds Failsafe integration tests (the `*IT.java` cases in `jjava-distro`, which spin up `eclipse-temurin` containers via Testcontainers). Testcontainers does not work on Windows; CI uses `mvn clean test` there. +- `mvn clean verify` — runs Failsafe integration tests (`*IT.java` in `jjava-distro`) via Testcontainers if Testcontainers can reach Docker (Windows GitHub-hosted runners cannot run WSL-backed containers). If Docker is unavailable, use `mvn clean test`. - `mvn test -pl jjava-kernel -am` — single-module unit tests (with required upstream modules built). - `mvn -pl jjava-kernel test -Dtest=JavaKernelExtensionsLifecycleTest` — run a single test class. - Surefire in `jjava-kernel` and `jjava-distro` is configured with `--add-opens jdk.jshell/jdk.jshell=ALL-UNNAMED` — required for JShell reflection. Anything that exercises `JavaKernel`/`CodeEvaluator` directly needs the same flag. - Local install of the freshly built kernel into Jupyter: see `DEV-GUIDE.md` (`unzip` the kernelspec zip, then `jupyter kernelspec install ... --name=java --user`). -Minimum toolchain: JDK 11 (`maven.compiler.release=11`). CI matrix builds against 11, 17, 21, 25. +Minimum toolchain: JDK 21 (`maven.compiler.release=21`). CI matrix builds against 21 and 25. ## Module Layout and Dependency Graph diff --git a/DEV-GUIDE.md b/DEV-GUIDE.md index 892a5a0..2c64483 100644 --- a/DEV-GUIDE.md +++ b/DEV-GUIDE.md @@ -22,7 +22,7 @@ You will need a **JDK** >= 11 and proper credentials for the `sonatype-central` Two submodules of this project have different release strategies: -- the library modules (`jjava-jupyter`, `jjava-kernel`, `jjava-maven`, `jjava-launcher`) are released +- the library modules (`jjava-jupyter`, `jjava-kernel`, `jjava-maven`) are released on Maven Central via the Sonatype Central Portal - the `jjava-distro` kernel assembly is released through GitHub Releases, and from there picked up by the [Homebrew tap](https://github.com/dflib/homebrew-tap) and PyPI diff --git a/README.md b/README.md index 3189b6b..7380214 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ _JJava is an evolution of the earlier [IJava kernel](https://github.com/SpencerP ## Requirements -1. Java 11 or newer +1. Java 21 or newer 2. Python and a Jupyter-like environment to use the kernel in. ## Installation diff --git a/jjava-distro/assembly/zipped-kernel.xml b/jjava-distro/assembly/zipped-kernel.xml index 9bb0369..12e9e3f 100644 --- a/jjava-distro/assembly/zipped-kernel.xml +++ b/jjava-distro/assembly/zipped-kernel.xml @@ -8,10 +8,6 @@ ${project.build.directory}/jjava-distro-${project.version}.jar jjava.jar - - ../jjava-launcher/${project.build.directory}/jjava-launcher-${project.version}.jar - jjava-launcher.jar - ../kernelspec/java/kernel.json diff --git a/jjava-distro/pom.xml b/jjava-distro/pom.xml index 72080a1..2c7e925 100644 --- a/jjava-distro/pom.xml +++ b/jjava-distro/pom.xml @@ -133,15 +133,6 @@ - - org.apache.maven.plugins - maven-surefire-plugin - - - --add-opens jdk.jshell/jdk.jshell=ALL-UNNAMED - - - org.apache.maven.plugins maven-assembly-plugin diff --git a/jjava-distro/src/main/java/org/dflib/jjava/distro/Env.java b/jjava-distro/src/main/java/org/dflib/jjava/distro/Env.java index 2d19ac8..2b8e75c 100644 --- a/jjava-distro/src/main/java/org/dflib/jjava/distro/Env.java +++ b/jjava-distro/src/main/java/org/dflib/jjava/distro/Env.java @@ -24,7 +24,6 @@ final class Env { public static final String JJAVA_STARTUP_SCRIPT = "JJAVA_STARTUP_SCRIPT"; public static final String JJAVA_LOAD_EXTENSIONS = "JJAVA_LOAD_EXTENSIONS"; - // not used by JJava, but rather by the kernel launcher script public static final String JJAVA_JVM_OPTS = "JJAVA_JVM_OPTS"; public static String timeout() { @@ -48,6 +47,11 @@ public static List compilerOpts() { return optsString != null ? Opts.splitOpts(optsString) : java.util.List.of(); } + public static List jvmOpts() { + String optsString = System.getenv(Env.JJAVA_JVM_OPTS); + return optsString != null ? Opts.splitOpts(optsString) : List.of(); + } + public static String extraClasspath() { return System.getenv(Env.JJAVA_CLASSPATH); } diff --git a/jjava-distro/src/main/java/org/dflib/jjava/distro/JJava.java b/jjava-distro/src/main/java/org/dflib/jjava/distro/JJava.java index 01059a8..0c92739 100644 --- a/jjava-distro/src/main/java/org/dflib/jjava/distro/JJava.java +++ b/jjava-distro/src/main/java/org/dflib/jjava/distro/JJava.java @@ -63,6 +63,7 @@ public static void main(String[] args) throws Exception { .extensionsEnabled(Env.extensionsEnabled()) .compilerOpts(Env.compilerOpts()) + .remoteVMOptions(Env.jvmOpts()) .timeout(timeout.time, timeout.timeUnit) .lineMagic("load", new LoadMagic()) diff --git a/jjava-distro/src/main/java/org/dflib/jjava/distro/NotebookInitializer.java b/jjava-distro/src/main/java/org/dflib/jjava/distro/NotebookInitializer.java index 5019f19..e00463f 100644 --- a/jjava-distro/src/main/java/org/dflib/jjava/distro/NotebookInitializer.java +++ b/jjava-distro/src/main/java/org/dflib/jjava/distro/NotebookInitializer.java @@ -2,17 +2,17 @@ import org.dflib.jjava.jupyter.Extension; import org.dflib.jjava.jupyter.kernel.BaseKernel; -import org.dflib.jjava.jupyter.kernel.BaseNotebookStatics; -import org.dflib.jjava.kernel.JavaNotebookStatics; +import org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics; +import org.dflib.jjava.kernel.ExecutorJavaNotebookStatics; /** * An automatically-loaded extension that adds common imports to the notebook. This includes static method imports - * from {@link BaseNotebookStatics} and {@link JavaNotebookStatics}. + * from the executor helper classes {@link ExecutorNotebookStatics} and {@link ExecutorJavaNotebookStatics}. */ public class NotebookInitializer implements Extension { - private static final String BASE_STATICS_CLASS = BaseNotebookStatics.class.getName(); - private static final String JAVA_STATICS_CLASS = JavaNotebookStatics.class.getName(); + private static final String BASE_STATICS_CLASS = ExecutorNotebookStatics.class.getName(); + private static final String JAVA_STATICS_CLASS = ExecutorJavaNotebookStatics.class.getName(); private static final String STARTUP_SCRIPT = "import java.util.*;\n" + "import java.io.*;\n" + diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/CellOutput.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/CellOutput.java index 91aa28f..7efa4cc 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/CellOutput.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/CellOutput.java @@ -87,7 +87,7 @@ public CellOutput assertError(String... errorFragments) { return this; } - private String field(String name) { + String field(String name) { return fields.getOrDefault(name, ""); } diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelEnvIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelEnvIT.java index 16bbcb7..1e0a7b0 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelEnvIT.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelEnvIT.java @@ -5,6 +5,7 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class KernelEnvIT extends ContainerizedKernelCase { @@ -25,9 +26,34 @@ public void compilerOpts() throws Exception { public void timeout() throws Exception { Map env = Map.of(Env.JJAVA_TIMEOUT, "3000"); String cell = "Thread.sleep(5000);"; - KernelRun run = executeInKernel(env, cell); + KernelRun run = executeInKernel(env, cell, "2 + 2"); run.cell(1).assertError("| " + cell, "Evaluation timed out after 3000 milliseconds."); + assertEquals("4", run.cell(2).result()); + } + + @Test + public void timeoutCoversTheWholeCell() throws Exception { + String cell = "Thread.sleep(2000); System.out.println(\"first complete\"); Thread.sleep(2000);"; + KernelRun run = executeInKernel(Map.of(Env.JJAVA_TIMEOUT, "3000"), "1 + 1", cell); + + assertEquals("2", run.cell(1).result()); + run.cell(2).assertError("Evaluation timed out after 3000 milliseconds."); + assertTrue(run.cell(2).stdout().contains("first complete"), run.cell(2).toString()); + } + + @Test + public void zeroTimeoutDisablesDeadline() throws Exception { + KernelRun run = executeInKernel(Map.of(Env.JJAVA_TIMEOUT, "0"), "Thread.sleep(100); 42").assertNoErrors(); + + assertEquals("42", run.cell(1).result()); + } + + @Test + public void negativeTimeoutDisablesDeadline() throws Exception { + KernelRun run = executeInKernel(Map.of(Env.JJAVA_TIMEOUT, "-1"), "Thread.sleep(100); 42").assertNoErrors(); + + assertEquals("42", run.cell(1).result()); } @Test @@ -38,7 +64,7 @@ public void classpath() throws Exception { "\"className = \" + Dummy.class.getName();" ).assertNoErrors(); - assertEquals("className = org.dflib.jjava.Dummy", run.cell(2).result()); + assertEquals("\"className = org.dflib.jjava.Dummy\"", run.cell(2).result()); } @Test @@ -46,7 +72,7 @@ public void startUpScriptsPath() throws Exception { Map env = Map.of(Env.JJAVA_STARTUP_SCRIPTS_PATH, CONTAINER_RESOURCES + "/test-ping.jshell"); KernelRun run = executeInKernel(env, "ping()").assertNoErrors(); - assertEquals("pong!", run.cell(1).result()); + assertEquals("\"pong!\"", run.cell(1).result()); } @Test @@ -54,7 +80,7 @@ public void startUpScript() throws Exception { Map env = Map.of(Env.JJAVA_STARTUP_SCRIPT, "public String ping() { return \"pong!\"; }"); KernelRun run = executeInKernel(env, "ping()").assertNoErrors(); - assertEquals("pong!", run.cell(1).result()); + assertEquals("\"pong!\"", run.cell(1).result()); } @Test diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java index 1c1d68a..790b5d5 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java @@ -3,6 +3,7 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class KernelExecutionIT extends ContainerizedKernelCase { @@ -22,7 +23,7 @@ public void variableSurvivesLaterImports() throws Exception { ) ).assertNoErrors(); - assertEquals("com.fasterxml.jackson.databind.ObjectMapper", run.cell(2).result()); + assertEquals("\"com.fasterxml.jackson.databind.ObjectMapper\"", run.cell(2).result()); } @Test @@ -45,7 +46,89 @@ public void variableRedeclarationInSameCell() throws Exception { "var v = \"b\";\nv" ).assertNoErrors(); - assertEquals("a", run.cell(1).result()); - assertEquals("b", run.cell(2).result()); + assertEquals("\"a\"", run.cell(1).result()); + assertEquals("\"b\"", run.cell(2).result()); + } + + @Test + public void textPlainStringsAndCustomObjects() throws Exception { + KernelRun run = executeInKernel( + "\"quoted\"", + "\"line1\\nline2\"", + "class DisplayValue { public String toString() { return \"custom-value\"; } }", + "new DisplayValue()" + ).assertNoErrors(); + + assertEquals("\"quoted\"", run.cell(1).result()); + assertEquals("\"line1\\nline2\"", run.cell(2).result()); + assertEquals("custom-value", run.cell(4).result()); + } + + @Test + public void methodRedefinitionKeepsSavedValues() throws Exception { + KernelRun run = executeInKernel( + "int calculate() { return 1; }", + "int saved = calculate();", + "int calculate() { return 2; }", + "saved", + "calculate()" + ).assertNoErrors(); + + assertEquals("1", run.cell(4).result()); + assertEquals("2", run.cell(5).result()); + } + + @Test + public void classRedefinitionKeepsSavedValuesAndInstances() throws Exception { + KernelRun run = executeInKernel( + "class Reloadable { int value() { return 1; } }", + "Reloadable instance = new Reloadable();", + "int saved = instance.value();", + "class Reloadable { int value() { return 2; } }", + "saved", + "instance.value()", + "new Reloadable().value()" + ).assertNoErrors(); + + assertEquals("1", run.cell(5).result()); + assertEquals("2", run.cell(6).result()); + assertEquals("2", run.cell(7).result()); + } + + @Test + public void interruptStopsBusyLoopAndKeepsExecutor() throws Exception { + KernelRun run = executeInKernel( + "@@INTERRUPT_AFTER_MS=500@@while (true) {}", + "1 + 1" + ); + + run.cell(1).assertError("Evaluation interrupted."); + assertTrue(Long.parseLong(run.cell(1).field("interruptLatencyMs")) <= 2000, "interrupt was not handled within two seconds"); + assertEquals("2", run.cell(2).result()); + } + + @Test + public void executorStreamsReachNotebook() throws Exception { + KernelRun run = executeInKernel( + "System.out.print(\"stdout-1\"); System.err.print(\"stderr-1\");", + "System.out.print(\"stdout-2\"); System.err.print(\"stderr-2\");" + ).assertNoErrors(); + + assertEquals("stdout-1", run.cell(1).stdout()); + assertEquals("stderr-1", run.cell(1).stderr()); + assertEquals("stdout-2", run.cell(2).stdout()); + assertEquals("stderr-2", run.cell(2).stderr()); + } + + @Test + public void userExceptionDoesNotLoseExecutorState() throws Exception { + KernelRun run = executeInKernel( + "int saved = 7;", + "throw new IllegalStateException(\"outer\", new IllegalArgumentException(\"inner\"));", + "saved + 1" + ); + + run.cell(2).assertError("IllegalStateException", "outer", "IllegalArgumentException", "inner", "at .(#"); + assertEquals("8", run.cell(3).result()); } } diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelMagicIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelMagicIT.java index 6554657..6398678 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelMagicIT.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelMagicIT.java @@ -1,5 +1,7 @@ package org.dflib.jjava.distro; +import com.google.gson.Gson; +import com.google.gson.JsonObject; import org.junit.jupiter.api.Test; import org.testcontainers.containers.Container; @@ -8,24 +10,23 @@ public class KernelMagicIT extends ContainerizedKernelCase { + private static final Gson GSON = new Gson(); + @Deprecated @Test public void jars() throws Exception { - String jar = CONTAINER_RESOURCES + "/jakarta.annotation-api-3.0.0.jar"; - Container.ExecResult fetchResult = container.execInContainer( - "curl", "-L", "-s", "-S", "-f", - "https://repo1.maven.org/maven2/jakarta/annotation/jakarta.annotation-api/3.0.0/jakarta.annotation-api-3.0.0.jar", - "-o", jar - ); - assertEquals(0, fetchResult.getExitCode(), fetchResult.getStdout()); + String jar = CONTAINER_RESOURCES + "/dummy.jar"; + Container.ExecResult jarResult = container.execInContainer( + "jar", "--create", "--file", jar, "-C", TEST_CLASSPATH, "."); + assertEquals(0, jarResult.getExitCode(), jarResult.getStderr() + "\n" + jarResult.getStdout()); KernelRun run = executeInKernel( "%jars " + jar, - "import jakarta.annotation.Nullable;", - "Nullable.class.getName()" + "import org.dflib.jjava.Dummy;", + "Dummy.class.getName()" ).assertNoErrors(); - assertEquals("jakarta.annotation.Nullable", run.cell(3).result()); + assertEquals("\"org.dflib.jjava.Dummy\"", run.cell(3).result()); } @Test @@ -36,18 +37,30 @@ public void classpath() throws Exception { "\"className = \" + Dummy.class.getName();" ).assertNoErrors(); - assertEquals("className = org.dflib.jjava.Dummy", run.cell(3).result()); + assertEquals("\"className = org.dflib.jjava.Dummy\"", run.cell(3).result()); } @Test public void maven() throws Exception { KernelRun run = executeInKernel( - "%maven org.dflib:dflib-jupyter:1.0.0-RC1", - "System.getProperty(\"java.class.path\")" + "%maven jakarta.annotation:jakarta.annotation-api:3.0.0", + "import jakarta.annotation.Nullable;", + "Nullable.class.getName()" ).assertNoErrors(); - String classpath = run.cell(2).result(); - assertTrue(classpath.contains("dflib-jupyter-1.0.0-RC1.jar"), () -> "Unexpected classpath: " + classpath); + assertEquals("\"jakarta.annotation.Nullable\"", run.cell(3).result()); + } + + @Test + public void executorKernelHelperFailsExplicitlyWithoutLosingState() throws Exception { + KernelRun run = executeInKernel( + "int saved = 40;", + "kernel();", + "saved + 2" + ); + + run.cell(2).assertError("JavaKernel.kernel() cannot expose the managing JVM"); + assertEquals("42", run.cell(3).result()); } @Deprecated @@ -55,11 +68,48 @@ public void maven() throws Exception { public void mavenIvySyntax() throws Exception { KernelRun run = executeInKernel( "%maven jakarta.annotation#jakarta.annotation-api;3.0.0", - "System.getProperty(\"java.class.path\")" + "import jakarta.annotation.Nullable;", + "Nullable.class.getName()" ).assertNoErrors(); - String classpath = run.cell(2).result(); - assertTrue(classpath.contains("jakarta.annotation-api-3.0.0.jar"), () -> "Unexpected classpath: " + classpath); + assertEquals("\"jakarta.annotation.Nullable\"", run.cell(3).result()); + } + + @Test + public void timeMagicCompletes() throws Exception { + KernelRun run = executeInKernel("%time 1 + 1").assertNoErrors(); + + assertEquals("2", run.cell(1).result()); + assertTrue(run.cell(1).display().contains("Wall time:"), run.cell(1).display()); + } + + @Test + public void portableDisplayBundlesReachTheManager() throws Exception { + String cell = String.join("\n", + "import org.dflib.jjava.jupyter.kernel.display.DisplayData;", + "var bundle = new DisplayData(\"portable\");", + "bundle.putHTML(\"portable\");", + "bundle.putData(\"image/png\", \"AQID\");", + "String id = display(bundle);" + ); + KernelRun run = executeInKernel(cell, "updateDisplay(id, new DisplayData(\"updated\"));").assertNoErrors(); + + JsonObject mime = GSON.fromJson(run.cell(1).field("mime"), JsonObject.class); + assertEquals("portable", mime.get("text/plain").getAsString()); + assertEquals("portable", mime.get("text/html").getAsString()); + assertEquals("AQID", mime.get("image/png").getAsString()); + assertEquals("updated", run.cell(2).display()); + } + + @Test + public void nonPortableDisplayPayloadFailsVisibly() throws Exception { + KernelRun run = executeInKernel( + "var data = new org.dflib.jjava.jupyter.kernel.display.DisplayData(); data.putData(\"text/html\", new Object()); display(data);", + "40 + 2" + ); + + run.cell(1).assertError("Non-portable display value"); + assertEquals("42", run.cell(2).result()); } @Test @@ -69,7 +119,7 @@ public void load() throws Exception { "ping()" ).assertNoErrors(); - assertEquals("pong!", run.cell(2).result()); + assertEquals("\"pong!\"", run.cell(2).result()); } @Test @@ -80,6 +130,6 @@ public void loadFromPOM() throws Exception { "Nullable.class.getName()" ).assertNoErrors(); - assertEquals("jakarta.annotation.Nullable", run.cell(3).result()); + assertEquals("\"jakarta.annotation.Nullable\"", run.cell(3).result()); } } diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java index eceea46..649bbe6 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java @@ -10,10 +10,62 @@ public class KernelStartupIT extends ContainerizedKernelCase { @Test - public void startUp() throws Exception { - KernelRun run = executeInKernel("1000d + 1").assertNoErrors(); + public void directStartupUsesOneExecutor() throws Exception { + KernelRun run = executeInKernel( + "ProcessHandle.current().pid()", + "ProcessHandle.current().pid()", + "ProcessHandle.current().parent().get().info().commandLine().orElse(\"\")" + ).assertNoErrors(); - assertEquals("1001.0", run.cell(1).result()); + assertEquals(run.cell(1).result(), run.cell(2).result()); + assertTrue(run.cell(3).result().contains("jjava.jar"), run.cell(3).result()); + } + + @Test + public void executorTerminationMarksSessionLost() throws Exception { + KernelRun run = executeInKernel( + Map.of(Env.JJAVA_TIMEOUT, "3000"), + "Runtime.getRuntime().halt(17);", + "1 + 1" + ); + + run.cell(1).assertError("Executor process terminated", "Restart the kernel"); + run.cell(2).assertError("Executor process terminated", "Restart the kernel"); + } + + @Test + public void displayAfterExecutorTerminationReportsExecutorUnavailable() throws Exception { + KernelRun run = executeInKernel( + Map.of(Env.JJAVA_TIMEOUT, "3000"), + "Runtime.getRuntime().halt(17);", + "display(\"after termination\")" + ); + + run.cell(1).assertError("Executor process terminated", "Restart the kernel"); + run.cell(2).assertError("Executor process terminated", "Restart the kernel"); + } + + @Test + public void executorOutOfMemoryMarksSessionLost() throws Exception { + KernelRun run = executeInKernel( + Map.of(Env.JJAVA_JVM_OPTS, "-Xmx128m"), + "new byte[Integer.MAX_VALUE]", + "1 + 1" + ); + + run.cell(1).assertError("Executor process terminated", "Restart the kernel"); + run.cell(2).assertError("Executor process terminated", "Restart the kernel"); + } + + @Test + public void notebookInitializerImportsAndHelpers() throws Exception { + KernelRun run = executeInKernel( + "LocalDate.of(2025, 1, 2).getYear()", + "printf(\"helper-%s\", \"ready\");" + ).assertNoErrors(); + + assertEquals("2025", run.cell(1).result()); + assertEquals("helper-ready", run.cell(2).stdout()); } @Test @@ -25,6 +77,6 @@ public void startUp_scriptRequiresClasspath() throws Exception { KernelRun run = executeInKernel(env, "\"hash = \" + obj.hashCode()").assertNoErrors(); String result = run.cell(1).result(); - assertTrue(result.matches("hash = -?\\d+"), () -> "Unexpected result: " + result); + assertTrue(result.matches("\\\"hash = -?\\d+\\\""), () -> "Unexpected result: " + result); } } diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/NotebookInitializerImportsTest.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/NotebookInitializerImportsTest.java new file mode 100644 index 0000000..8d5afcf --- /dev/null +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/NotebookInitializerImportsTest.java @@ -0,0 +1,23 @@ +package org.dflib.jjava.distro; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class NotebookInitializerImportsTest { + + @Test + public void startupScriptImportsOnlyExecutorHelpers() throws Exception { + Field field = NotebookInitializer.class.getDeclaredField("STARTUP_SCRIPT"); + field.setAccessible(true); + String script = (String) field.get(null); + + assertTrue(script.contains("import static org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics.*;"), script); + assertTrue(script.contains("import static org.dflib.jjava.kernel.ExecutorJavaNotebookStatics.*;"), script); + assertFalse(script.contains("BaseNotebookStatics"), script); + assertFalse(script.contains("org.dflib.jjava.kernel.JavaNotebookStatics"), script); + } +} diff --git a/jjava-distro/src/test/resources/kernel-driver.py b/jjava-distro/src/test/resources/kernel-driver.py index e574ed3..b53932c 100644 --- a/jjava-distro/src/test/resources/kernel-driver.py +++ b/jjava-distro/src/test/resources/kernel-driver.py @@ -15,8 +15,11 @@ """ import base64 +import json import re import sys +import threading +import time from queue import Empty from jupyter_client.manager import start_new_kernel @@ -47,31 +50,50 @@ def receive(receiver, channel, cell_number): % (cell_number, channel, MESSAGE_TIMEOUT)) -def execute(client, cell_number, source): +def execute(client, manager, cell_number, source): + interrupt_match = re.match(r"^@@INTERRUPT_AFTER_MS=(\d+)@@", source) + interrupt_after_ms = int(interrupt_match.group(1)) if interrupt_match else None + if interrupt_match: + source = source[interrupt_match.end():].lstrip("\n") + + started = time.monotonic() msg_id = client.execute(source, allow_stdin=False) + timer = None + if interrupt_after_ms is not None: + timer = threading.Timer(interrupt_after_ms / 1000, manager.interrupt_kernel) + timer.daemon = True + timer.start() - outputs = {"result": "", "display": "", "stdout": "", "stderr": "", "error": ""} - while True: - message = receive(client.get_iopub_msg, "iopub", cell_number) - if message["parent_header"].get("msg_id") != msg_id: - continue - - message_type = message["msg_type"] - content = message["content"] - if message_type == "status": - if content["execution_state"] == "idle": - break - elif message_type == "stream": - key = "stdout" if content["name"] == "stdout" else "stderr" - outputs[key] += content["text"] - elif message_type == "execute_result": - outputs["result"] += content["data"].get("text/plain", "") - elif message_type in ("display_data", "update_display_data"): - outputs["display"] += content["data"].get("text/plain", "") - elif message_type == "error": - lines = ["%s: %s" % (content["ename"], content["evalue"])] - lines.extend(content["traceback"]) - outputs["error"] += ANSI.sub("", "\n".join(lines)) + outputs = {"result": "", "display": "", "mime": "", "stdout": "", "stderr": "", "error": ""} + try: + while True: + message = receive(client.get_iopub_msg, "iopub", cell_number) + if message["parent_header"].get("msg_id") != msg_id: + continue + + message_type = message["msg_type"] + content = message["content"] + if message_type == "status": + if content["execution_state"] == "idle": + break + elif message_type == "stream": + key = "stdout" if content["name"] == "stdout" else "stderr" + outputs[key] += content["text"] + elif message_type == "execute_result": + outputs["result"] += content["data"].get("text/plain", "") + elif message_type in ("display_data", "update_display_data"): + outputs["display"] += content["data"].get("text/plain", "") + outputs["mime"] += json.dumps(content["data"], separators=(",", ":")) + elif message_type == "error": + lines = ["%s: %s" % (content["ename"], content["evalue"])] + lines.extend(content["traceback"]) + outputs["error"] += ANSI.sub("", "\n".join(lines)) + finally: + if timer: + timer.cancel() + + if interrupt_after_ms is not None: + outputs["interruptLatencyMs"] = str(max(0, int((time.monotonic() - started) * 1000) - interrupt_after_ms)) # "wait_for_ready" re-sends "kernel_info_request" until the kernel answers, and consumes # only one of the replies, so the shell channel may still hold replies to those extra @@ -90,7 +112,7 @@ def main(cells): manager, client = start_new_kernel(kernel_name="java", startup_timeout=STARTUP_TIMEOUT) try: for i, source in enumerate(cells): - execute(client, i + 1, source) + execute(client, manager, i + 1, source) finally: client.stop_channels() manager.shutdown_kernel() diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java index 8acdf85..9d0c99d 100644 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java @@ -7,8 +7,8 @@ import java.util.UUID; /** - * A collection of static methods for notebook code to interact with the kernel. The methods are automatically exposed - * in notebooks via a static import on bootstrap. + * Manager-side notebook helpers. They always use the running kernel directly; executor code uses + * {@link ExecutorNotebookStatics} instead. */ public class BaseNotebookStatics { @@ -22,7 +22,6 @@ public static Object eval(String source) { public static T lineMagic(String name, List args) { BaseKernel kernel = BaseKernel.notebookKernel(); - try { return kernel.getMagicsRegistry().evalLineMagic(kernel, name, args); } catch (UndefinedMagicException e) { @@ -34,7 +33,6 @@ public static T lineMagic(String name, List args) { public static T cellMagic(String name, List args, String body) { BaseKernel kernel = BaseKernel.notebookKernel(); - try { return kernel.getMagicsRegistry().evalCellMagic(kernel, name, args, body); } catch (UndefinedMagicException e) { @@ -53,39 +51,32 @@ public static DisplayData render(Object o, String... as) { } public static String display(Object o) { + return displayData(render(o)); + } - DisplayData data = render(o); - - String id = data.getDisplayId(); - if (id == null) { - id = UUID.randomUUID().toString(); - data.setDisplayId(id); - } + public static String display(Object o, String... as) { + return displayData(render(o, as)); + } - BaseKernel.notebookKernel().display(data); - return id; + public static void updateDisplay(String id, Object o) { + updateDisplayData(id, render(o)); } - public static String display(Object o, String... as) { - DisplayData data = render(o, as); + public static void updateDisplay(String id, Object o, String... as) { + updateDisplayData(id, render(o, as)); + } + private static String displayData(DisplayData data) { String id = data.getDisplayId(); if (id == null) { id = UUID.randomUUID().toString(); data.setDisplayId(id); } - BaseKernel.notebookKernel().display(data); return id; } - public static void updateDisplay(String id, Object o) { - DisplayData data = render(o); - BaseKernel.notebookKernel().getIO().display.updateDisplay(id, data); - } - - public static void updateDisplay(String id, Object o, String... as) { - DisplayData data = render(o, as); + private static void updateDisplayData(String id, DisplayData data) { BaseKernel.notebookKernel().getIO().display.updateDisplay(id, data); } } diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStatics.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStatics.java new file mode 100644 index 0000000..94dd89d --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStatics.java @@ -0,0 +1,100 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryBootstrap; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; + +import java.util.List; +import java.util.UUID; + +/** + * Notebook helpers for executor (user cell) code. Fixed by import at wiring time: display requests go through + * {@link DisplayDeliveryBootstrap}, rendering is local, and the kernel is never read. + */ +public class ExecutorNotebookStatics { + + private ExecutorNotebookStatics() { + } + + public static void printf(String format, Object... args) { + System.out.printf(format, args); + } + + public static Object eval(String source) { + throw new UnsupportedOperationException("eval() cannot be called from executor code in JDI mode"); + } + + public static T lineMagic(String name, List args) { + throw new UnsupportedOperationException( + "Magics cannot be called programmatically from executor code in JDI mode; use %magic syntax"); + } + + public static T cellMagic(String name, List args, String body) { + throw new UnsupportedOperationException( + "Magics cannot be called programmatically from executor code in JDI mode; use %%magic syntax"); + } + + public static DisplayData render(Object o) { + return renderPortable(o); + } + + @SuppressWarnings("unused") + public static DisplayData render(Object o, String... as) { + return renderPortable(o); + } + + public static String display(Object o) { + return displayData(render(o)); + } + + public static String display(Object o, String... as) { + return displayData(render(o, as)); + } + + public static void updateDisplay(String id, Object o) { + updateDisplayData(id, render(o)); + } + + public static void updateDisplay(String id, Object o, String... as) { + updateDisplayData(id, render(o, as)); + } + + private static String displayData(DisplayData data) { + String id = data.getDisplayId(); + if (id == null) { + id = UUID.randomUUID().toString(); + data.setDisplayId(id); + } + deliver(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, id, data)); + return id; + } + + private static void updateDisplayData(String id, DisplayData data) { + deliver(DisplayRequest.of(DisplayRequest.Operation.UPDATE, id, data)); + } + + // one attempt, no retry; messages keep the existing prefixes + private static void deliver(DisplayRequest request) { + try { + DisplayDeliveryBootstrap.current().deliver(request); + } catch (DisplayDeliveryException e) { + switch (e.kind()) { + case UNAVAILABLE -> throw new IllegalStateException("Remote display channel is not available", e); + case REJECTED -> throw new RuntimeException(e.getMessage(), e); + case UNCERTAIN -> throw new RuntimeException("Failed to send display data: " + e.getMessage(), e); + } + throw e; + } + } + + private static DisplayData renderPortable(Object value) { + if (value instanceof DisplayData data) { + return data; + } + if (value instanceof CharSequence || value instanceof Number || value instanceof Boolean) { + return new DisplayData(String.valueOf(value)); + } + throw new IllegalArgumentException("Non-portable display value: " + value.getClass().getName()); + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/DisplayData.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/DisplayData.java index ed0ff3b..d31eb7f 100644 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/DisplayData.java +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/DisplayData.java @@ -96,6 +96,18 @@ public boolean hasDataForType(MIMEType type) { return this.data.containsKey(type.toString()); } + public Map getData() { + return Collections.unmodifiableMap(data); + } + + public Map getMetadata() { + return Collections.unmodifiableMap(metadata); + } + + public Map getTransientData() { + return transientData == null ? Collections.emptyMap() : Collections.unmodifiableMap(transientData); + } + public void assign(DisplayData data) { this.data.putAll(data.data); diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDelivery.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDelivery.java new file mode 100644 index 0000000..0152354 --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDelivery.java @@ -0,0 +1,10 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +/** + * Executor-side boundary: hands one display request to the manager and returns only on confirmed delivery, otherwise + * throws a {@link DisplayDeliveryException}. + */ +public interface DisplayDelivery { + + void deliver(DisplayRequest request); +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrap.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrap.java new file mode 100644 index 0000000..9c28bb8 --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrap.java @@ -0,0 +1,46 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayDelivery; + +import java.nio.file.Path; +import java.util.Objects; + +/** + * Chooses the executor-side delivery from the {@code jjava.display.socket} system property. Without the property, + * every delivery fails as UNAVAILABLE. + */ +public final class DisplayDeliveryBootstrap { + + public static final String SOCKET_PROPERTY = "jjava.display.socket"; + + private static DisplayDelivery current; + + private DisplayDeliveryBootstrap() { + } + + public static synchronized DisplayDelivery current() { + if (current == null) { + current = fromSocketPath(System.getProperty(SOCKET_PROPERTY)); + } + return current; + } + + /** + * Replaces the delivery and returns the previous one. + */ + public static synchronized DisplayDelivery install(DisplayDelivery delivery) { + DisplayDelivery previous = current(); + current = Objects.requireNonNull(delivery); + return previous; + } + + static DisplayDelivery fromSocketPath(String socketPath) { + if (socketPath == null) { + return request -> { + throw new DisplayDeliveryException( + DisplayDeliveryException.Kind.UNAVAILABLE, "Remote display channel is not available", null); + }; + } + return new UnixDomainDisplayDelivery(Path.of(socketPath)); + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryException.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryException.java new file mode 100644 index 0000000..c2e4cb1 --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryException.java @@ -0,0 +1,21 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +public class DisplayDeliveryException extends IllegalStateException { + + public enum Kind { + UNAVAILABLE, + REJECTED, + UNCERTAIN + } + + private final Kind kind; + + public DisplayDeliveryException(Kind kind, String message, Throwable cause) { + super(message, cause); + this.kind = kind; + } + + public Kind kind() { + return kind; + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocol.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocol.java new file mode 100644 index 0000000..18f5dd0 --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocol.java @@ -0,0 +1,208 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.ToNumberPolicy; +import org.dflib.jjava.jupyter.kernel.display.DisplayData; + +import java.lang.reflect.Array; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; + +/** + * Transport-independent codec and portability rules of the cross-JVM display protocol. A request is one JSON object; + * a response is one token: {@code ok}, or a JSON string with the failure message. Nothing here depends on how the + * bytes travel. + */ +public final class DisplayProtocol { + + static final Gson GSON = new GsonBuilder() + .serializeNulls() + .setObjectToNumberStrategy(ToNumberPolicy.LAZILY_PARSED_NUMBER) + .create(); + + private static final String OK = "ok"; + private static final String ACTION = "action"; + private static final String TRANSIENT = "transient"; + + private DisplayProtocol() { + } + + static JsonObject snapshot(DisplayRequest.Operation operation, String displayId, DisplayData data) { + if (displayId == null) { + throw new IllegalArgumentException(operation == DisplayRequest.Operation.UPDATE + ? "updateDisplay requires a non-null id" + : "display requires a non-null id"); + } + + Set path = Collections.newSetFromMap(new IdentityHashMap()); + requirePortable(data.getData(), path); + requirePortable(data.getMetadata(), path); + requirePortable(data.getTransientData(), path); + + JsonObject bundle = new JsonObject(); + bundle.add("data", GSON.toJsonTree(data.getData())); + bundle.add("metadata", GSON.toJsonTree(data.getMetadata())); + bundle.add(TRANSIENT, GSON.toJsonTree(data.getTransientData())); + if (operation == DisplayRequest.Operation.DISPLAY) { + bundle.getAsJsonObject(TRANSIENT).addProperty("display_id", displayId); + } + return bundle; + } + + public static String encode(DisplayRequest request) { + JsonObject message = new JsonObject(); + if (request.operation() == DisplayRequest.Operation.UPDATE) { + message.addProperty(ACTION, "updateDisplay"); + message.addProperty("id", request.displayId()); + } else { + message.addProperty(ACTION, "display"); + } + message.add("bundle", request.bundle()); + return GSON.toJson(message); + } + + public static DisplayRequest decode(String line) { + JsonObject message; + try { + message = GSON.fromJson(line, JsonObject.class); + } catch (JsonParseException e) { + throw new IllegalArgumentException("Display request must be a JSON object", e); + } + if (message == null) { + throw new IllegalArgumentException("Display request must be a JSON object"); + } + + JsonElement bundle = message.get("bundle"); + if (bundle == null || !bundle.isJsonObject()) { + throw new IllegalArgumentException("Display bundle must not be null"); + } + + JsonElement action = message.get(ACTION); + String name = action != null && action.isJsonPrimitive() ? action.getAsString() : null; + if ("display".equals(name)) { + return new DisplayRequest( + DisplayRequest.Operation.DISPLAY, + transientDisplayId(bundle.getAsJsonObject()), + bundle.getAsJsonObject()); + } + if ("updateDisplay".equals(name)) { + JsonElement id = message.get("id"); + if (id == null || id.isJsonNull()) { + throw new IllegalArgumentException("updateDisplay requires a non-null id"); + } + return new DisplayRequest(DisplayRequest.Operation.UPDATE, id.getAsString(), bundle.getAsJsonObject()); + } + throw new IllegalArgumentException("Unsupported display action: " + name); + } + + /** + * Returns the response token for a handled request: {@code null} means success. + */ + public static String response(RuntimeException failure) { + if (failure == null) { + return OK; + } + String message = failure.getMessage() == null ? failure.getClass().getSimpleName() : failure.getMessage(); + return GSON.toJson(message); + } + + /** + * Interprets a response token; throws unless it is {@code ok}. + */ + public static void checkResponse(String response) { + if (response == null) { + throw new DisplayDeliveryException( + DisplayDeliveryException.Kind.UNCERTAIN, "Display callback closed without a response", null); + } + if (OK.equals(response)) { + return; + } + + String message; + try { + message = GSON.fromJson(response, String.class); + } catch (JsonParseException e) { + message = response; + } + throw new DisplayDeliveryException( + DisplayDeliveryException.Kind.REJECTED, + "Display callback failed: " + (message == null ? response : message), + null); + } + + private static String transientDisplayId(JsonObject bundle) { + JsonElement transientData = bundle.get(TRANSIENT); + JsonElement id = transientData != null && transientData.isJsonObject() + ? transientData.getAsJsonObject().get("display_id") + : null; + return id != null && id.isJsonPrimitive() ? id.getAsString() : null; + } + + private static void requirePortable(Object value, Set path) { + if (value == null || value instanceof CharSequence || value instanceof Boolean || value instanceof Character) { + return; + } + if (value instanceof Number number) { + requireFinite(number); + return; + } + + enter(value, path); + requireContainer(value, path); + path.remove(value); + } + + private static void requireContainer(Object value, Set path) { + if (value instanceof Map map) { + for (Map.Entry entry : map.entrySet()) { + requireStringKey(entry.getKey()); + requirePortable(entry.getValue(), path); + } + } else if (value instanceof Iterable items) { + for (Object item : items) { + requirePortable(item, path); + } + } else if (value.getClass().isArray()) { + for (int i = 0; i < Array.getLength(value); i++) { + requirePortable(Array.get(value, i), path); + } + } else { + throw new IllegalArgumentException("Non-portable display value: " + value.getClass().getName()); + } + } + + private static void requireStringKey(Object key) { + if (!(key instanceof String)) { + throw new IllegalArgumentException("Non-portable display value: " + + (key == null ? "null" : key.getClass().getName())); + } + } + + private static void requireFinite(Number number) { + if (isNonFinite(number)) { + throw new IllegalArgumentException("Non-portable display value: non-finite number"); + } + } + + private static void enter(Object container, Set path) { + if (!path.add(container)) { + throw new IllegalArgumentException("Non-portable display value: cyclic structure"); + } + } + + private static boolean isNonFinite(Number number) { + if (number instanceof Double d) { + return !Double.isFinite(d); + } + if (number instanceof Float f) { + return !Float.isFinite(f); + } + return false; + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequest.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequest.java new file mode 100644 index 0000000..7c12fed --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequest.java @@ -0,0 +1,40 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.dflib.jjava.jupyter.kernel.display.DisplayData; + +import java.util.Map; +import java.util.function.BiConsumer; + +public record DisplayRequest(Operation operation, String displayId, JsonObject bundle) { + + public enum Operation { + DISPLAY, + UPDATE + } + + public static DisplayRequest of(Operation operation, String displayId, DisplayData data) { + return new DisplayRequest(operation, displayId, DisplayProtocol.snapshot(operation, displayId, data)); + } + + public DisplayData toDisplayData() { + DisplayData out = new DisplayData(); + copy("data", out::putData); + copy("metadata", out::putMetaData); + copy("transient", out::putTransientData); + if (displayId != null) { + out.setDisplayId(displayId); + } + return out; + } + + private void copy(String section, BiConsumer sink) { + JsonElement values = bundle.get(section); + if (values != null && values.isJsonObject()) { + for (Map.Entry entry : values.getAsJsonObject().entrySet()) { + sink.accept(entry.getKey(), DisplayProtocol.GSON.fromJson(entry.getValue(), Object.class)); + } + } + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequestHandler.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequestHandler.java new file mode 100644 index 0000000..1f3a468 --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequestHandler.java @@ -0,0 +1,10 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +/** + * Manager-side boundary: applies one decoded display request to the notebook. Any exception it throws is reported to + * the executor as a rejection. + */ +public interface DisplayRequestHandler { + + void handle(DisplayRequest request); +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayDelivery.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayDelivery.java new file mode 100644 index 0000000..817a173 --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayDelivery.java @@ -0,0 +1,71 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol.uds; + +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDelivery; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayProtocol; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; + +import java.io.IOException; +import java.net.StandardProtocolFamily; +import java.net.UnixDomainSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import static org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException.Kind.UNAVAILABLE; +import static org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException.Kind.UNCERTAIN; +import static org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint.DEADLINES; +import static org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint.DEADLINE_SECONDS; +import static org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint.closeQuietly; +import static org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint.readLine; + +/** + * Executor-side delivery over a Unix-domain socket. One deadline covers connect, write, and the response read. A + * failure before any payload byte is written is UNAVAILABLE; a failure after is UNCERTAIN. Nothing is retried. + */ +public final class UnixDomainDisplayDelivery implements DisplayDelivery { + + private static final String UNAVAILABLE_MESSAGE = "Remote display channel is not available"; + private static final String UNCERTAIN_MESSAGE = "Display delivery outcome is uncertain; nothing was retried"; + + private final Path socketPath; + + public UnixDomainDisplayDelivery(Path socketPath) { + this.socketPath = socketPath; + } + + @Override + public void deliver(DisplayRequest request) { + ByteBuffer out = ByteBuffer.wrap((DisplayProtocol.encode(request) + "\n").getBytes(StandardCharsets.UTF_8)); + + SocketChannel channel; + try { + channel = SocketChannel.open(StandardProtocolFamily.UNIX); + } catch (IOException e) { + throw new DisplayDeliveryException(UNAVAILABLE, UNAVAILABLE_MESSAGE, e); + } + + ScheduledFuture deadline = DEADLINES.schedule(() -> closeQuietly(channel), DEADLINE_SECONDS, TimeUnit.SECONDS); + // Once connected, a write may have partly reached the peer even if the call never returned (a deadline close + // interrupts it, and the byte count only advances on return). So UNAVAILABLE is reserved for failures before connect. + boolean connected = false; + try (channel) { + channel.connect(UnixDomainSocketAddress.of(socketPath)); + connected = true; + while (out.hasRemaining()) { + channel.write(out); + } + DisplayProtocol.checkResponse(readLine(channel)); + } catch (IOException e) { + throw new DisplayDeliveryException( + connected ? UNCERTAIN : UNAVAILABLE, + connected ? UNCERTAIN_MESSAGE : UNAVAILABLE_MESSAGE, + e); + } finally { + deadline.cancel(false); + } + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayEndpoint.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayEndpoint.java new file mode 100644 index 0000000..57a260a --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayEndpoint.java @@ -0,0 +1,158 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol.uds; + +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayProtocol; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequestHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.IOException; +import java.net.StandardProtocolFamily; +import java.net.UnixDomainSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * Manager-side Unix-domain endpoint. Accepts one request line per connection, serially, and answers with one response + * line. Each connection is bounded by a 10 s deadline. Closing the endpoint, or an accept failure, releases the + * channel and removes the socket file. + */ +public final class UnixDomainDisplayEndpoint implements Closeable { + + static final long DEADLINE_SECONDS = 10; + static final ScheduledExecutorService DEADLINES = Executors.newSingleThreadScheduledExecutor(r -> { + Thread thread = new Thread(r, "jjava-display-deadlines"); + thread.setDaemon(true); + return thread; + }); + + private static final Logger LOGGER = LoggerFactory.getLogger(UnixDomainDisplayEndpoint.class); + + private final ServerSocketChannel server; + private final Path path; + private final DisplayRequestHandler handler; + + public UnixDomainDisplayEndpoint(DisplayRequestHandler handler) throws IOException { + this(null, handler); + } + + public UnixDomainDisplayEndpoint(Path socketPath, DisplayRequestHandler handler) throws IOException { + this.handler = handler; + this.server = ServerSocketChannel.open(StandardProtocolFamily.UNIX); + Path boundPath = null; + try { + this.server.bind(socketPath == null ? null : UnixDomainSocketAddress.of(socketPath)); + boundPath = socketPath; + boundPath = ((UnixDomainSocketAddress) server.getLocalAddress()).getPath(); + this.path = boundPath; + + Thread acceptor = new Thread(this::acceptLoop, "jjava-display-callback"); + acceptor.setDaemon(true); + acceptor.start(); + } catch (IOException | RuntimeException | Error e) { + closeQuietly(server); + if (boundPath != null) { + try { + Files.deleteIfExists(boundPath); + } catch (IOException cleanup) { + e.addSuppressed(cleanup); + } + } + throw e; + } + } + + public Path path() { + return path; + } + + private void acceptLoop() { + while (true) { + SocketChannel channel; + try { + channel = server.accept(); + } catch (IOException e) { + if (server.isOpen()) { + LOGGER.warn("Display endpoint accept failed; closing endpoint", e); + closeQuietly(this); + } + return; + } + try { + handle(channel); + } finally { + closeQuietly(channel); + } + } + } + + private void handle(SocketChannel channel) { + ScheduledFuture deadline = DEADLINES.schedule(() -> closeQuietly(channel), DEADLINE_SECONDS, TimeUnit.SECONDS); + try { + writeLine(channel, respond(channel)); + } catch (IOException e) { + LOGGER.warn("Display connection failed", e); + } finally { + deadline.cancel(false); + } + } + + private String respond(SocketChannel channel) throws IOException { + try { + handler.handle(DisplayProtocol.decode(readLine(channel))); + return DisplayProtocol.response(null); + } catch (RuntimeException e) { + return DisplayProtocol.response(e); + } + } + + @Override + public void close() throws IOException { + server.close(); + Files.deleteIfExists(path); + } + + static String readLine(ReadableByteChannel channel) throws IOException { + ByteArrayOutputStream line = new ByteArrayOutputStream(); + ByteBuffer buffer = ByteBuffer.allocate(8192); + while (true) { + buffer.clear(); + if (channel.read(buffer) < 0) { + return line.size() == 0 ? null : line.toString(StandardCharsets.UTF_8); + } + buffer.flip(); + while (buffer.hasRemaining()) { + byte b = buffer.get(); + if (b == '\n') { + return line.toString(StandardCharsets.UTF_8); + } + line.write(b); + } + } + } + + static void writeLine(SocketChannel channel, String text) throws IOException { + ByteBuffer out = ByteBuffer.wrap((text + "\n").getBytes(StandardCharsets.UTF_8)); + while (out.hasRemaining()) { + channel.write(out); + } + } + + static void closeQuietly(Closeable closeable) { + try { + closeable.close(); + } catch (IOException ignored) { + // best-effort close on a failure or deadline path; the caller has no recovery action + } + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspiler.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspiler.java index 74d294c..1791ffe 100644 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspiler.java +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspiler.java @@ -1,6 +1,6 @@ package org.dflib.jjava.jupyter.kernel.magic; -import org.dflib.jjava.jupyter.kernel.BaseNotebookStatics; +import org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics; import java.util.Base64; import java.util.regex.Matcher; @@ -15,8 +15,8 @@ public class MagicTranspiler { private static final Pattern UNESCAPED_QUOTE = Pattern.compile("(? split = split(matchedLine.group("args")); diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/TelemetryCollector.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/TelemetryCollector.java deleted file mode 100644 index 93f1439..0000000 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/TelemetryCollector.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.dflib.jjava.jupyter.telemetry; - -/** - * Aggregates some telemetry number for a sequence of "measurements". Instead of a single method wrapping a measure code - * invocation lambda, each measurement is done via two separate methods: {@link #measurementStart()} and - * {@link #measurementEnd(Object)}. This is to make the collector minimally invasive and not adding to the call stack - * trace. - */ -public interface TelemetryCollector { - - I measurementStart(); - - void measurementEnd(I startState); - - void stop(); - - TelemetryCollector DO_NOTHING = new TelemetryCollector<>() { - @Override - public void measurementEnd(Object startState) { - } - - @Override - public Object measurementStart() { - return null; - } - - @Override - public void stop() { - } - }; -} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallAndCpuTimeCollector.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallAndCpuTimeCollector.java deleted file mode 100644 index 346b8a1..0000000 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallAndCpuTimeCollector.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.dflib.jjava.jupyter.telemetry; - -import java.lang.management.ThreadMXBean; -import java.util.concurrent.atomic.AtomicLong; - -public abstract class WallAndCpuTimeCollector implements TelemetryCollector { - - private final ThreadMXBean threadMXBean; - private final AtomicLong totalWT; - private final AtomicLong totalUT; - private final AtomicLong totalCT; - - public static boolean canMeasureCpuTimes(ThreadMXBean threadMXBean) { - // according to JavaDocs, isThreadCpuTimeEnabled() can throw in some cases - try { - return threadMXBean.isCurrentThreadCpuTimeSupported() && threadMXBean.isThreadCpuTimeEnabled(); - } catch (Exception e) { - return false; - } - } - - public WallAndCpuTimeCollector(ThreadMXBean threadMXBean) { - this.threadMXBean = threadMXBean; - this.totalWT = new AtomicLong(); - this.totalUT = new AtomicLong(); - this.totalCT = new AtomicLong(); - } - - @Override - public Measure measurementStart() { - return new Measure( - System.nanoTime(), - threadMXBean.getCurrentThreadUserTime(), - threadMXBean.getCurrentThreadCpuTime() - ); - } - - @Override - public void measurementEnd(Measure m) { - - long wt = System.nanoTime(); - long ut = threadMXBean.getCurrentThreadUserTime(); - long ct = threadMXBean.getCurrentThreadCpuTime(); - - totalWT.addAndGet(wt - m.wt); - - // sanity check - if (m.ut != -1 && ut != -1 && m.ct != -1 && ct != -1) { - totalUT.addAndGet(ut - m.ut); - totalCT.addAndGet(ct - m.ct); - } else { - totalUT.set(-1); - totalCT.set(-1); - } - } - - @Override - public void stop() { - onStop(totalWT.get(), totalUT.get(), totalCT.get()); - } - - protected abstract void onStop(long wallTimeNanos, long userTimeNanos, long totalTimeNanos); - - public static class Measure { - final long wt; - final long ut; - final long ct; - - public Measure(long wt, long ut, long ct) { - this.ct = ct; - this.wt = wt; - this.ut = ut; - } - } -} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallTimeCollector.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallTimeCollector.java deleted file mode 100644 index 403bac9..0000000 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallTimeCollector.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.dflib.jjava.jupyter.telemetry; - -import java.util.concurrent.atomic.AtomicLong; - -public abstract class WallTimeCollector implements TelemetryCollector { - - private final AtomicLong totalWallTime; - - public WallTimeCollector() { - this.totalWallTime = new AtomicLong(); - } - - @Override - public Long measurementStart() { - return System.nanoTime(); - } - - @Override - public void measurementEnd(Long startState) { - long delta = System.nanoTime() - startState; - totalWallTime.addAndGet(delta); - } - - @Override - public void stop() { - onStop(totalWallTime.get()); - } - - protected abstract void onStop(long wallTimeNanos); -} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStaticsDisplayTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStaticsDisplayTest.java new file mode 100644 index 0000000..2e60651 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStaticsDisplayTest.java @@ -0,0 +1,131 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.dflib.jjava.jupyter.channels.ShellReplyEnvironment; +import org.dflib.jjava.jupyter.kernel.comm.CommManager; +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.Renderer; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDelivery; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryBootstrap; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; +import org.dflib.jjava.jupyter.kernel.magic.MagicTranspiler; +import org.dflib.jjava.jupyter.kernel.magic.MagicsRegistry; +import org.dflib.jjava.jupyter.kernel.magic.MagicsResolver; +import org.dflib.jjava.jupyter.kernel.util.StringStyler; +import org.dflib.jjava.jupyter.messages.Message; +import org.dflib.jjava.jupyter.messages.publish.PublishDisplayData; +import org.dflib.jjava.jupyter.messages.publish.PublishUpdateDisplayData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Manager-side helpers: display and update go to the running kernel directly and never through DisplayDelivery. + */ +public class BaseNotebookStaticsDisplayTest { + + private final List delivered = new ArrayList<>(); + private BaseKernel previousKernel; + private DisplayDelivery previousDelivery; + private CapturingEnvironment env; + + @BeforeEach + public void setUp() { + previousKernel = BaseKernel.notebookKernel; + previousDelivery = DisplayDeliveryBootstrap.install(delivered::add); + + JupyterIO io = new JupyterIO(StandardCharsets.UTF_8); + env = new CapturingEnvironment(); + io.setEnv(env); + BaseKernel.notebookKernel = new StubKernel(io); + } + + @AfterEach + void tearDown() { + BaseKernel.notebookKernel = previousKernel; + DisplayDeliveryBootstrap.install(previousDelivery); + } + + @Test + void displayReturnsTheIdOfThePublishedBundle() { + DisplayData data = new DisplayData("x"); + data.setDisplayId("display-id"); + + String id = BaseNotebookStatics.display(data); + + PublishDisplayData published = assertInstanceOf(PublishDisplayData.class, env.published.get(0).getContent()); + assertEquals(published.getDisplayId(), id); + assertTrue(delivered.isEmpty()); + } + + @Test + public void updateDisplayUsesKernelWithTheTargetId() { + BaseNotebookStatics.updateDisplay("requested-id", new DisplayData("updated")); + + PublishUpdateDisplayData update = assertInstanceOf( + PublishUpdateDisplayData.class, + env.published.get(0).getContent()); + assertEquals("requested-id", update.getDisplayId()); + assertTrue(delivered.isEmpty()); + } + + @Test + public void managerWithoutKernelFailsWithNullPointerAndNeverDelivers() { + BaseKernel.notebookKernel = null; + + assertThrows(NullPointerException.class, () -> BaseNotebookStatics.display("text")); + assertTrue(delivered.isEmpty()); + } + + private static class CapturingEnvironment extends ShellReplyEnvironment { + final List> published = new ArrayList<>(); + + CapturingEnvironment() { + super(null, null, null, null); + } + + @Override + public void publish(Message message) { + published.add(message); + } + } + + private static class StubKernel extends BaseKernel { + StubKernel(JupyterIO io) { + super( + "test", "0", + new LanguageInfo.Builder("Java").build(), + List.of(), null, io, new CommManager(), new Renderer(), + new MagicsResolver("%", "%%", new MagicTranspiler()), + new MagicsRegistry(), false, new StringStyler.Builder().build()); + } + + @Override + protected Object doEval(String source) { + throw new UnsupportedOperationException(); + } + + @Override + public DisplayData inspect(String code, int at, boolean extraDetail) { + throw new UnsupportedOperationException(); + } + + @Override + public ReplacementOptions complete(String code, int at) { + throw new UnsupportedOperationException(); + } + + @Override + public String isComplete(String code) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsFailureTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsFailureTest.java new file mode 100644 index 0000000..6fbae95 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsFailureTest.java @@ -0,0 +1,70 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDelivery; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryBootstrap; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ExecutorNotebookStaticsFailureTest { + + private DisplayDelivery previousDelivery; + private AtomicInteger attempts; + + @BeforeEach + void setUp() { + attempts = new AtomicInteger(); + previousDelivery = null; + } + + @AfterEach + void tearDown() { + if (previousDelivery != null) { + DisplayDeliveryBootstrap.install(previousDelivery); + } + } + + @Test + public void unavailableMapsToRemoteChannelMessage() { + failingWith(DisplayDeliveryException.Kind.UNAVAILABLE, "Remote display channel is not available"); + + RuntimeException e = assertThrows(RuntimeException.class, () -> ExecutorNotebookStatics.display("text")); + + assertEquals("Remote display channel is not available", e.getMessage()); + assertEquals(1, attempts.get()); + } + + @Test + public void rejectedKeepsDisplayCallbackPrefix() { + failingWith(DisplayDeliveryException.Kind.REJECTED, "Display callback failed: boom"); + + RuntimeException e = assertThrows(RuntimeException.class, () -> ExecutorNotebookStatics.display("text")); + + assertTrue(e.getMessage().startsWith("Display callback failed: "), e.getMessage()); + assertEquals(1, attempts.get()); + } + + @Test + public void uncertainMapsToFailedToSendPrefixAndIsNotRetried() { + failingWith(DisplayDeliveryException.Kind.UNCERTAIN, "outcome unknown"); + + RuntimeException e = assertThrows(RuntimeException.class, () -> ExecutorNotebookStatics.updateDisplay("id", "text")); + + assertTrue(e.getMessage().startsWith("Failed to send display data: "), e.getMessage()); + assertEquals(1, attempts.get()); + } + + private void failingWith(DisplayDeliveryException.Kind kind, String message) { + previousDelivery = DisplayDeliveryBootstrap.install(request -> { + attempts.incrementAndGet(); + throw new DisplayDeliveryException(kind, message, null); + }); + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRenderCharacterizationTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRenderCharacterizationTest.java new file mode 100644 index 0000000..2005140 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRenderCharacterizationTest.java @@ -0,0 +1,79 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDelivery; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryBootstrap; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Characterizes executor render behaviour (Gate 1 for the routing change). Rendering never publishes or delivers, so + * the recording delivery must stay empty throughout. + */ +public class ExecutorNotebookStaticsRenderCharacterizationTest { + + private final List delivered = new ArrayList<>(); + private DisplayDelivery previousDelivery; + private BaseKernel previousKernel; + + @BeforeEach + void setUp() { + previousKernel = BaseKernel.notebookKernel; + BaseKernel.notebookKernel = null; + previousDelivery = DisplayDeliveryBootstrap.install(delivered::add); + } + + @AfterEach + void tearDown() { + DisplayDeliveryBootstrap.install(previousDelivery); + BaseKernel.notebookKernel = previousKernel; + } + + @Test + public void renderNullThrowsNullPointerException() { + assertThrows(NullPointerException.class, () -> ExecutorNotebookStatics.render((Object) null)); + } + + @Test + public void renderDisplayDataReturnsSameObject() { + DisplayData data = new DisplayData("x"); + assertSame(data, ExecutorNotebookStatics.render((Object) data)); + } + + @Test + public void renderScalarsAsTextPlainOnly() { + assertEquals(Map.of("text/plain", "hi"), ExecutorNotebookStatics.render("hi").getData()); + assertEquals(Map.of("text/plain", "42"), ExecutorNotebookStatics.render(42).getData()); + assertEquals(Map.of("text/plain", "true"), ExecutorNotebookStatics.render(true).getData()); + } + + @Test + public void renderCharacterThrowsIllegalArgumentException() { + assertThrows(IllegalArgumentException.class, () -> ExecutorNotebookStatics.render('c')); + } + + @Test + public void renderIgnoresPresentationHints() { + assertEquals( + ExecutorNotebookStatics.render("x").getData(), + ExecutorNotebookStatics.render("x", "text/html").getData()); + } + + @Test + public void renderNeverDelivers() { + ExecutorNotebookStatics.render("x"); + ExecutorNotebookStatics.render("x", "text/html"); + assertTrue(delivered.isEmpty()); + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRoutingTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRoutingTest.java new file mode 100644 index 0000000..fbbfbd4 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRoutingTest.java @@ -0,0 +1,90 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDelivery; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryBootstrap; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ExecutorNotebookStaticsRoutingTest { + + private final List delivered = new ArrayList<>(); + private DisplayDelivery previousDelivery; + private BaseKernel previousKernel; + + @BeforeEach + void setUp() { + previousKernel = BaseKernel.notebookKernel; + BaseKernel.notebookKernel = null; + previousDelivery = DisplayDeliveryBootstrap.install(delivered::add); + } + + @AfterEach + void tearDown() { + DisplayDeliveryBootstrap.install(previousDelivery); + BaseKernel.notebookKernel = previousKernel; + } + + @Test + public void displayDeliversThroughBootstrapWithoutReadingKernel() { + String id = ExecutorNotebookStatics.display("text"); + + assertEquals(1, delivered.size()); + assertEquals(DisplayRequest.Operation.DISPLAY, delivered.get(0).operation()); + assertEquals(id, delivered.get(0).displayId()); + } + + @Test + public void updateDisplayDeliversUpdateWithTargetId() { + ExecutorNotebookStatics.updateDisplay("target", "text"); + + assertEquals(DisplayRequest.Operation.UPDATE, delivered.get(0).operation()); + assertEquals("target", delivered.get(0).displayId()); + } + + @Test + public void updateDisplayWithNullIdFailsBeforeDelivering() { + assertThrows(IllegalArgumentException.class, () -> ExecutorNotebookStatics.updateDisplay(null, "text")); + assertTrue(delivered.isEmpty()); + } + + @Test + public void unsupportedHelpersFailWithExistingMessages() { + assertEquals( + "eval() cannot be called from executor code in JDI mode", + assertThrows(UnsupportedOperationException.class, () -> ExecutorNotebookStatics.eval("1")).getMessage()); + assertEquals( + "Magics cannot be called programmatically from executor code in JDI mode; use %magic syntax", + assertThrows(UnsupportedOperationException.class, + () -> ExecutorNotebookStatics.lineMagic("m", List.of())).getMessage()); + assertEquals( + "Magics cannot be called programmatically from executor code in JDI mode; use %%magic syntax", + assertThrows(UnsupportedOperationException.class, + () -> ExecutorNotebookStatics.cellMagic("m", List.of(), "body")).getMessage()); + assertTrue(delivered.isEmpty()); + } + + @Test + public void printfWritesStandardOutput() { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, StandardCharsets.UTF_8)); + try { + ExecutorNotebookStatics.printf("x=%d", 7); + } finally { + System.setOut(original); + } + assertEquals("x=7", captured.toString(StandardCharsets.UTF_8)); + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/OutputBoundaryDependencyTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/OutputBoundaryDependencyTest.java new file mode 100644 index 0000000..aae893b --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/OutputBoundaryDependencyTest.java @@ -0,0 +1,81 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.spi.ToolProvider; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class OutputBoundaryDependencyTest { + + private static final String PROTOCOL = "org.dflib.jjava.jupyter.kernel.display.protocol."; + private static final String BASE_STATICS = "org.dflib.jjava.jupyter.kernel.BaseNotebookStatics"; + + @Test + public void protocolClassesOutsideTransportPackagesDoNotUseSocketsOrJupyterChannels() { + List violations = new ArrayList<>(); + List edges = edges(Path.of("target", "classes")); + assertFalse(edges.isEmpty(), "jdeps produced no edges; the check would pass vacuously"); + for (String[] edge : edges) { + String from = edge[0]; + String to = edge[1]; + if (isDirectProtocolClass(from) && (to.startsWith("java.net.") + || to.startsWith("java.nio.channels.") + || to.startsWith("org.dflib.jjava.jupyter.channels.") + || to.startsWith("org.dflib.jjava.jupyter.messages.") + || to.equals("org.dflib.jjava.jupyter.kernel.DisplayStream") + || to.equals("org.dflib.jjava.jupyter.kernel.BaseKernel"))) { + violations.add(from + " -> " + to); + } + } + assertTrue(violations.isEmpty(), "forbidden edges: " + violations); + } + + @Test + public void managerHelperNeverDeliversThroughTheExecutorBoundary() { + List violations = new ArrayList<>(); + for (String[] edge : edges(Path.of("target", "classes"))) { + String from = edge[0]; + String to = edge[1]; + if (from.startsWith(BASE_STATICS) && (to.startsWith(PROTOCOL + "uds.") + || to.startsWith(PROTOCOL + "tcp.") + || to.equals(PROTOCOL + "DisplayDelivery") + || to.equals(PROTOCOL + "DisplayDeliveryBootstrap") + || to.startsWith("java.net."))) { + violations.add(from + " -> " + to); + } + } + assertTrue(violations.isEmpty(), "forbidden edges: " + violations); + } + + // A class directly in the protocol package (not in uds/tcp subpackages). + private static boolean isDirectProtocolClass(String name) { + return name.startsWith(PROTOCOL) && name.indexOf('.', PROTOCOL.length()) < 0; + } + + static List edges(Path classes) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ToolProvider.findFirst("jdeps").orElseThrow().run( + new PrintStream(out, true, StandardCharsets.UTF_8), System.err, + "-verbose:class", classes.toString()); + + List edges = new ArrayList<>(); + for (String line : out.toString(StandardCharsets.UTF_8).split("\n")) { + int arrow = line.indexOf(" -> "); + if (arrow < 0) { + continue; + } + String from = line.substring(0, arrow).trim(); + String to = line.substring(arrow + 4).trim().split("\\s+")[0]; + edges.add(new String[]{from, to}); + } + return edges; + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrapTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrapTest.java new file mode 100644 index 0000000..067ea0c --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrapTest.java @@ -0,0 +1,44 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayDelivery; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException.Kind.UNAVAILABLE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class DisplayDeliveryBootstrapTest { + + private static final DisplayRequest REQUEST = DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", new DisplayData("x")); + + @Test + public void noSocketPropertyFailsEveryDeliveryUnavailable() { + DisplayDelivery unavailable = DisplayDeliveryBootstrap.fromSocketPath(null); + DisplayDeliveryException e = assertThrows(DisplayDeliveryException.class, () -> unavailable.deliver(REQUEST)); + assertEquals(UNAVAILABLE, e.kind()); + } + + @Test + public void socketPropertySelectsUdsDelivery() { + assertInstanceOf(UnixDomainDisplayDelivery.class, DisplayDeliveryBootstrap.fromSocketPath("/tmp/x.sock")); + } + + @Test + public void installReturnsPreviousAndIsUsedForSubsequentCalls() { + DisplayDelivery previous = DisplayDeliveryBootstrap.current(); + List seen = new ArrayList<>(); + try { + assertSame(previous, DisplayDeliveryBootstrap.install(seen::add)); + DisplayDeliveryBootstrap.current().deliver(REQUEST); + assertEquals(1, seen.size()); + } finally { + DisplayDeliveryBootstrap.install(previous); + } + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryContractTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryContractTest.java new file mode 100644 index 0000000..473c59f --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryContractTest.java @@ -0,0 +1,100 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException.Kind.REJECTED; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Contract every delivery must satisfy, shared by the transports and the in-memory substitute. Transport-specific + * behaviour (unavailable endpoint, stalled peer) is tested in the transport's own test classes. + */ +public abstract class DisplayDeliveryContractTest { + + private static final Gson GSON = new GsonBuilder().serializeNulls().create(); + + protected abstract DisplayDelivery delivery(DisplayRequestHandler handler); + + @Test + public void bundleSurvivesDelivery() { + List received = new ArrayList<>(); + DisplayData sent = bundle(); + + delivery(received::add).deliver(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id-1", sent)); + + DisplayRequest request = received.get(0); + assertEquals("id-1", request.displayId()); + DisplayData got = request.toDisplayData(); + assertEquals(GSON.toJsonTree(sent.getData()), GSON.toJsonTree(got.getData())); + assertEquals(GSON.toJsonTree(sent.getMetadata()), GSON.toJsonTree(got.getMetadata())); + assertEquals("id-1", got.getDisplayId()); + } + + @Test + public void updateCarriesTargetId() { + List received = new ArrayList<>(); + DisplayData data = new DisplayData("updated"); + data.putTransientData("display_id", "other"); + + delivery(received::add).deliver(DisplayRequest.of(DisplayRequest.Operation.UPDATE, "target", data)); + + DisplayRequest request = received.get(0); + assertEquals(DisplayRequest.Operation.UPDATE, request.operation()); + assertEquals("target", request.displayId()); + assertEquals("target", request.toDisplayData().getDisplayId()); + } + + @Test + public void sequentialOperationsKeepOrder() { + List ids = new ArrayList<>(); + DisplayDelivery delivery = delivery(r -> ids.add(r.displayId())); + + List expected = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + String id = String.valueOf(i); + expected.add(id); + delivery.deliver(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, id, new DisplayData(id))); + } + assertEquals(expected, ids); + } + + @Test + public void handlerFailureIsRejectedWithItsMessage() { + DisplayDelivery delivery = delivery(r -> { + throw new IllegalStateException("boom"); + }); + DisplayRequest request = DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "x", new DisplayData("t")); + + DisplayDeliveryException e = assertThrows( + DisplayDeliveryException.class, + () -> delivery.deliver(request)); + assertEquals(REJECTED, e.kind()); + assertTrue(e.getMessage().contains("boom")); + } + + private static DisplayData bundle() { + DisplayData data = new DisplayData("portable"); + data.putHTML("portable"); + data.putData("image/png", "AQID"); + + Map json = new LinkedHashMap<>(); + json.put("values", Arrays.asList("kept", null)); + json.put("nothing", null); + data.putData("application/json", json); + + data.putMetaData("render", Map.of("expanded", "yes")); + data.putTransientData("custom", "preserved"); + return data; + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java new file mode 100644 index 0000000..1b812bf --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java @@ -0,0 +1,61 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.Base64; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +public class DisplayEncodingTest { + + @Test + public void nestedJsonNumbersSurviveRoundTripWithoutPrecisionLoss() { + long integer = 9007199254740993L; + BigDecimal decimal = new BigDecimal("12345678901234567890.123456789012345678901234567890"); + DisplayData data = new DisplayData(); + data.putData("application/json", Map.of("integer", integer, "decimal", decimal)); + + String line = DisplayProtocol.encode(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); + DisplayData decoded = DisplayProtocol.decode(line).toDisplayData(); + Map numbers = (Map) decoded.getData().get("application/json"); + + assertEquals(Long.toString(integer), ((Number) numbers.get("integer")).toString()); + assertEquals(decimal.toString(), ((Number) numbers.get("decimal")).toString()); + + String reencoded = DisplayProtocol.encode(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", decoded)); + var json = DisplayProtocol.decode(reencoded).bundle().getAsJsonObject("data") + .getAsJsonObject("application/json"); + assertEquals(Long.toString(integer), json.get("integer").getAsNumber().toString()); + assertEquals(decimal.toString(), json.get("decimal").getAsNumber().toString()); + } + + @Test + public void multilineNonAsciiAndPngSurviveRoundTripUnchanged() { + String multiline = "line one\nline two\r\nend"; + String nonAscii = "héllo — 日本語 🎉"; + String png = Base64.getEncoder().encodeToString(new byte[]{(byte) 0x89, 'P', 'N', 'G', 0x0A, 0x1A, 0x0A}); + + DisplayData data = new DisplayData(multiline); + data.putData("text/x-non-ascii", nonAscii); + data.putData("image/png", png); + + String line = DisplayProtocol.encode(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); + DisplayData decoded = DisplayProtocol.decode(line).toDisplayData(); + + assertEquals(multiline, decoded.getData().get("text/plain")); + assertEquals(nonAscii, decoded.getData().get("text/x-non-ascii")); + assertEquals(png, decoded.getData().get("image/png")); + } + + @Test + public void encodedRequestNeverContainsRawLineFeed() { + DisplayData data = new DisplayData("a\nb\nc"); + String line = DisplayProtocol.encode(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); + + assertFalse(line.contains("\n"), line); + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocolTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocolTest.java new file mode 100644 index 0000000..294d460 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocolTest.java @@ -0,0 +1,87 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class DisplayProtocolTest { + + @Test + public void roundTripsDisplayAndUpdate() { + DisplayData data = new DisplayData("hello"); + data.putHTML("hi"); + + DisplayRequest display = DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id-1", data); + DisplayRequest decodedDisplay = DisplayProtocol.decode(DisplayProtocol.encode(display)); + assertEquals(DisplayRequest.Operation.DISPLAY, decodedDisplay.operation()); + assertEquals("id-1", decodedDisplay.displayId()); + assertEquals("hi", decodedDisplay.toDisplayData().getData().get("text/html")); + + DisplayRequest update = DisplayRequest.of(DisplayRequest.Operation.UPDATE, "id-1", data); + DisplayRequest decodedUpdate = DisplayProtocol.decode(DisplayProtocol.encode(update)); + assertEquals(DisplayRequest.Operation.UPDATE, decodedUpdate.operation()); + assertEquals("id-1", decodedUpdate.displayId()); + } + + @Test + public void rejectsCyclicStructure() { + List cycle = new ArrayList<>(); + cycle.add(cycle); + DisplayData data = new DisplayData(); + data.putData("application/json", cycle); + + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); + assertTrue(e.getMessage().contains("cyclic structure")); + } + + @Test + public void rejectsNonFiniteNumbers() { + for (double value : new double[]{Double.NaN, Double.POSITIVE_INFINITY}) { + DisplayData data = new DisplayData(); + data.putData("application/json", value); + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); + assertTrue(e.getMessage().contains("non-finite number")); + } + } + + @Test + public void rejectsNonStringMapKeys() { + Map map = new LinkedHashMap<>(); + map.put(1, "bad"); + DisplayData data = new DisplayData(); + data.putData("application/json", map); + + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); + assertTrue(e.getMessage().contains("Non-portable display value")); + } + + @Test + public void rejectsUpdateWithoutId() { + DisplayData data = new DisplayData("x"); + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> DisplayRequest.of(DisplayRequest.Operation.UPDATE, null, data)); + assertEquals("updateDisplay requires a non-null id", e.getMessage()); + } + + @Test + public void rejectsUnknownActionAndMalformedJson() { + assertThrows(IllegalArgumentException.class, + () -> DisplayProtocol.decode("{\"action\":\"explode\",\"bundle\":{}}")); + assertThrows(IllegalArgumentException.class, () -> DisplayProtocol.decode("not json")); + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/InMemoryDisplayDeliveryContractTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/InMemoryDisplayDeliveryContractTest.java new file mode 100644 index 0000000..8351e92 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/InMemoryDisplayDeliveryContractTest.java @@ -0,0 +1,23 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +/** + * Runs the delivery contract against an in-memory substitute that round-trips every request and response through the + * protocol text, exactly as a real transport would. + */ +public class InMemoryDisplayDeliveryContractTest extends DisplayDeliveryContractTest { + + @Override + protected DisplayDelivery delivery(DisplayRequestHandler handler) { + return request -> { + DisplayRequest received = DisplayProtocol.decode(DisplayProtocol.encode(request)); + String response; + try { + handler.handle(received); + response = DisplayProtocol.response(null); + } catch (RuntimeException e) { + response = DisplayProtocol.response(e); + } + DisplayProtocol.checkResponse(response); + }; + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryContractTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryContractTest.java new file mode 100644 index 0000000..09b9465 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryContractTest.java @@ -0,0 +1,33 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayDelivery; +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint; +import org.junit.jupiter.api.AfterEach; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; + +public class UnixDomainDisplayDeliveryContractTest extends DisplayDeliveryContractTest { + + private final List endpoints = new ArrayList<>(); + + @Override + protected DisplayDelivery delivery(DisplayRequestHandler handler) { + try { + UnixDomainDisplayEndpoint endpoint = new UnixDomainDisplayEndpoint(handler); + endpoints.add(endpoint); + return new UnixDomainDisplayDelivery(endpoint.path()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @AfterEach + void closeEndpoints() throws IOException { + for (UnixDomainDisplayEndpoint endpoint : endpoints) { + endpoint.close(); + } + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java new file mode 100644 index 0000000..fa54db1 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java @@ -0,0 +1,100 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayDelivery; +import org.junit.jupiter.api.Test; + +import java.net.StandardProtocolFamily; +import java.net.UnixDomainSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException.Kind.UNAVAILABLE; +import static org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException.Kind.UNCERTAIN; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Production bounds: a single 10 s RESPONSE deadline, judged with a 5 s scheduling tolerance (15 s). + */ +public class UnixDomainDisplayDeliveryTimeoutTest { + + private static final long BOUND_MILLIS = 15_000; + + @Test + public void missingSocketFailsUnavailableWithinConnectBound() { + Path missing = Path.of(System.getProperty("java.io.tmpdir"), "jjava-missing-" + System.nanoTime() + ".sock"); + UnixDomainDisplayDelivery delivery = new UnixDomainDisplayDelivery(missing); + DisplayRequest request = request(new DisplayData("x")); + long start = System.nanoTime(); + + DisplayDeliveryException e = assertThrows(DisplayDeliveryException.class, () -> delivery.deliver(request)); + + assertEquals(UNAVAILABLE, e.kind()); + assertTrue(elapsedMillis(start) < BOUND_MILLIS); + } + + @Test + public void peerThatNeverReadsLargeRequestFailsUncertainWithinBound() throws Exception { + Path socket = Files.createTempDirectory("uds-timeout").resolve("s.sock"); + try (ServerSocketChannel server = listen(socket)) { + // Never accepts: connect completes through the listen backlog, then the large write stalls. + UnixDomainDisplayDelivery delivery = new UnixDomainDisplayDelivery(socket); + DisplayRequest request = request(new DisplayData("x".repeat(8 << 20))); + long start = System.nanoTime(); + + DisplayDeliveryException e = assertThrows(DisplayDeliveryException.class, () -> delivery.deliver(request)); + + // the cause is reported so a platform-specific failure mode is visible in CI output + assertEquals(UNCERTAIN, e.kind(), () -> "cause: " + e.getCause()); + assertTrue(elapsedMillis(start) < BOUND_MILLIS); + } + } + + @Test + public void peerThatReadsButNeverRepliesFailsUncertainWithinBound() throws Exception { + Path socket = Files.createTempDirectory("uds-timeout").resolve("s.sock"); + try (ServerSocketChannel server = listen(socket)) { + Thread peer = new Thread(() -> holdAfterRead(server)); + peer.setDaemon(true); + peer.start(); + + UnixDomainDisplayDelivery delivery = new UnixDomainDisplayDelivery(socket); + DisplayRequest request = request(new DisplayData("small")); + long start = System.nanoTime(); + + DisplayDeliveryException e = assertThrows(DisplayDeliveryException.class, () -> delivery.deliver(request)); + + assertEquals(UNCERTAIN, e.kind()); + assertTrue(elapsedMillis(start) < BOUND_MILLIS); + } + } + + // Reads the request, then blocks until the client closes its end at the deadline (read returns -1). + private static void holdAfterRead(ServerSocketChannel server) { + try (SocketChannel accepted = server.accept()) { + accepted.read(ByteBuffer.allocate(1 << 20)); + accepted.read(ByteBuffer.allocate(1)); + } catch (Exception ignored) { + // the peer thread only exists to stall the client; its own failure has no observer + } + } + + private static ServerSocketChannel listen(Path socket) throws Exception { + ServerSocketChannel server = ServerSocketChannel.open(StandardProtocolFamily.UNIX); + server.bind(UnixDomainSocketAddress.of(socket)); + return server; + } + + private static DisplayRequest request(DisplayData data) { + return DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data); + } + + private static long elapsedMillis(long startNanos) { + return (System.nanoTime() - startNanos) / 1_000_000; + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayEndpointTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayEndpointTest.java new file mode 100644 index 0000000..04f560e --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayEndpointTest.java @@ -0,0 +1,59 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.StandardProtocolFamily; +import java.net.UnixDomainSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class UnixDomainDisplayEndpointTest { + + @Test + public void servesOneRequestLinePerConnectionInOrderAndRemovesSocketOnShutdown() throws Exception { + List ids = Collections.synchronizedList(new ArrayList<>()); + UnixDomainDisplayEndpoint endpoint = new UnixDomainDisplayEndpoint(r -> ids.add(r.displayId())); + Path socket = endpoint.path(); + try { + assertEquals("ok", send(socket, line("first"))); + assertEquals("ok", send(socket, line("second"))); + assertEquals(List.of("first", "second"), ids); + assertTrue(send(socket, "not json").startsWith("\"")); + } finally { + endpoint.close(); + } + assertFalse(Files.exists(socket)); + } + + private static String line(String id) { + return DisplayProtocol.encode(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, id, new DisplayData("x"))); + } + + private static String send(Path socket, String line) throws IOException { + try (SocketChannel channel = SocketChannel.open(StandardProtocolFamily.UNIX)) { + channel.connect(UnixDomainSocketAddress.of(socket)); + channel.write(ByteBuffer.wrap((line + "\n").getBytes(StandardCharsets.UTF_8))); + + ByteArrayOutputStream response = new ByteArrayOutputStream(); + ByteBuffer one = ByteBuffer.allocate(1); + while (channel.read(one.clear()) > 0 && one.get(0) != '\n') { + response.write(one.get(0)); + } + return response.toString(StandardCharsets.UTF_8); + } + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspilerTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspilerTest.java new file mode 100644 index 0000000..3a1396b --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspilerTest.java @@ -0,0 +1,26 @@ +package org.dflib.jjava.jupyter.kernel.magic; + +import org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class MagicTranspilerTest { + + private static final String EXECUTOR = ExecutorNotebookStatics.class.getName(); + + @Test + public void generatedCellAndLineCallsTargetExecutorHelpers() { + MagicTranspiler transpiler = new MagicTranspiler(); + + String cell = transpiler.transpileCell(new ParsedCellMagic("m", List.of(), "body")); + String line = transpiler.transpileLine(new ParsedLineMagic("m", List.of(), "x = ", "%m")); + + assertTrue(cell.startsWith(EXECUTOR + ".cellMagic("), cell); + assertTrue(line.startsWith(EXECUTOR + ".lineMagic("), line); + assertFalse(cell.contains("BaseNotebookStatics") || line.contains("BaseNotebookStatics")); + } +} diff --git a/jjava-kernel/pom.xml b/jjava-kernel/pom.xml index c54c2ab..cee0a59 100644 --- a/jjava-kernel/pom.xml +++ b/jjava-kernel/pom.xml @@ -37,7 +37,6 @@ org.apache.maven.plugins maven-surefire-plugin - --add-opens jdk.jshell/jdk.jshell=ALL-UNNAMED false diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/ExecutorJavaNotebookStatics.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/ExecutorJavaNotebookStatics.java new file mode 100644 index 0000000..9b25b35 --- /dev/null +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/ExecutorJavaNotebookStatics.java @@ -0,0 +1,22 @@ +package org.dflib.jjava.kernel; + +/** + * Kernel access for executor (user cell) code. Always fails; the managing JVM's kernel is never exposed to user code. + */ +public class ExecutorJavaNotebookStatics { + + private ExecutorJavaNotebookStatics() { + } + + /** + * @deprecated in favor of {@link #kernel()} + */ + @Deprecated(since = "1.0", forRemoval = true) + public static JavaKernel getKernelInstance() { + return kernel(); + } + + public static JavaKernel kernel() { + throw new UnsupportedOperationException("JavaKernel.kernel() cannot expose the managing JVM from executor code"); + } +} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java index ac5e2b9..8e1cc39 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java @@ -8,6 +8,7 @@ import jdk.jshell.SourceCodeAnalysis; import jdk.jshell.UnresolvedReferenceException; import org.dflib.jjava.jupyter.kernel.BaseKernel; +import org.dflib.jjava.jupyter.kernel.EvalBuilder; import org.dflib.jjava.jupyter.kernel.HelpLink; import org.dflib.jjava.jupyter.kernel.JupyterIO; import org.dflib.jjava.jupyter.kernel.LanguageInfo; @@ -17,6 +18,9 @@ import org.dflib.jjava.jupyter.kernel.display.Renderer; import org.dflib.jjava.jupyter.kernel.history.HistoryManager; import org.dflib.jjava.jupyter.kernel.magic.MagicTranspiler; +import org.dflib.jjava.jupyter.kernel.magic.ParsedCellMagic; +import org.dflib.jjava.jupyter.kernel.magic.ParsedLineMagic; +import org.dflib.jjava.jupyter.kernel.magic.UndefinedMagicException; import org.dflib.jjava.jupyter.kernel.magic.MagicsRegistry; import org.dflib.jjava.jupyter.kernel.magic.MagicsResolver; import org.dflib.jjava.jupyter.kernel.util.CharPredicate; @@ -25,10 +29,16 @@ import org.dflib.jjava.kernel.execution.CodeEvaluator; import org.dflib.jjava.kernel.execution.CompilationException; import org.dflib.jjava.kernel.execution.EvaluationInterruptedException; +import org.dflib.jjava.kernel.execution.ExecutorTerminationException; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequestHandler; +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint; +import org.dflib.jjava.kernel.execution.EvaluationResult; import org.dflib.jjava.kernel.execution.EvaluationTimeoutException; import org.dflib.jjava.kernel.execution.IncompleteSourceException; -import java.nio.charset.Charset; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -61,6 +71,8 @@ public static Builder builder() { private final JShell jShell; private final CodeEvaluator evaluator; + private volatile boolean shuttingDown; + private UnixDomainDisplayEndpoint displayEndpoint; protected JavaKernel( String name, @@ -94,6 +106,9 @@ protected JavaKernel( this.jShell = jShell; this.evaluator = evaluator; + jShell.onShutdown(ignored -> { + if (!shuttingDown) evaluator.markExecutorTerminated(); + }); } /** @@ -127,6 +142,8 @@ protected List formatError(Throwable e) { return formatEvaluationTimeoutException((EvaluationTimeoutException) e); } else if (e instanceof EvaluationInterruptedException) { return formatEvaluationInterruptedException((EvaluationInterruptedException) e); + } else if (e instanceof ExecutorTerminationException) { + return List.of(errorStyler.secondary(e.getMessage())); } else if (e instanceof RuntimeException && e.getCause() instanceof EvalException) { return formatEvalException((EvalException) e.getCause()); } else { @@ -241,11 +258,109 @@ private List formatEvaluationInterruptedException(EvaluationInterruptedE return fmt; } + @Override + public EvalBuilder evalBuilder(String source) { + return new JavaEvalBuilder<>(this, source, null, null, false); + } + @Override protected Object doEval(String source) { return evaluator.eval(jShell, source); } + /** + * Maps an evaluation or magic result to its Jupyter presentation: absent evaluation text produces no output, + * evaluation text becomes text/plain only, and any other magic value is rendered as text. + */ + static DisplayData toExecuteResult(Object value) { + if (value instanceof EvaluationResult result) { + String text = result.text(); + return text == null ? null : new DisplayData(text); + } + if (value == null || value instanceof DisplayData) { + return (DisplayData) value; + } + return new DisplayData(String.valueOf(value)); + } + + private static final class JavaEvalBuilder implements EvalBuilder { + private final JavaKernel kernel; + private final String source; + private final ParsedLineMagic lineMagic; + private final ParsedCellMagic cellMagic; + private final boolean renderResults; + + private JavaEvalBuilder( + JavaKernel kernel, + String source, + ParsedLineMagic lineMagic, + ParsedCellMagic cellMagic, + boolean renderResults) { + this.kernel = kernel; + this.source = source; + this.lineMagic = lineMagic; + this.cellMagic = cellMagic; + this.renderResults = renderResults; + } + + @Override + public EvalBuilder resolveMagics() { + if (lineMagic != null || cellMagic != null) { + return this; + } + + ParsedCellMagic cell = kernel.getMagicsResolver().parseCellMagic(source); + if (cell != null) { + return new JavaEvalBuilder<>(kernel, source, null, cell, renderResults); + } + + ParsedLineMagic line = kernel.getMagicsResolver().parseLineMagic(source); + if (line != null && line.magicLinePrefix.isBlank()) { + return new JavaEvalBuilder<>(kernel, source, line, null, renderResults); + } + + return new JavaEvalBuilder<>(kernel, kernel.getMagicsResolver().resolve(source), null, null, renderResults); + } + + @SuppressWarnings("unchecked") + @Override + public EvalBuilder renderResults() { + return renderResults + ? (EvalBuilder) this + : new JavaEvalBuilder<>(kernel, source, lineMagic, cellMagic, true); + } + + @Override + @SuppressWarnings("unchecked") + public T eval() { + Object value = evalRaw(); + if (!renderResults) { + return (T) (value instanceof EvaluationResult result ? result.text() : value); + } + return (T) toExecuteResult(value); + } + + private Object evalRaw() { + if (cellMagic == null && lineMagic == null) { + return kernel.doEval(source); + } + + try { + return cellMagic != null + ? kernel.getMagicsRegistry().evalCellMagic(kernel, cellMagic.name, cellMagic.args, cellMagic.cellBodyAfterMagic) + : kernel.getMagicsRegistry().evalLineMagic(kernel, lineMagic.name, lineMagic.args); + } catch (UndefinedMagicException e) { + throw e; + } catch (Exception e) { + String magicName = cellMagic != null ? cellMagic.name : lineMagic.name; + String kind = cellMagic != null ? "cell" : "line"; + throw new RuntimeException( + String.format("Exception running %s magic '%s': %s", kind, magicName, e.getMessage()), + e); + } + } + } + @Override public DisplayData inspect(String code, int at, boolean extraDetail) { // Move the code position to the end of the identifier to make the inspection work at any @@ -350,21 +465,36 @@ public String isComplete(String code) { @Override public void onShutdown(boolean isRestarting) { - super.onShutdown(isRestarting); - jShell.close(); + shuttingDown = true; + try { + super.onShutdown(isRestarting); + } finally { + try { + jShell.close(); + } finally { + try { + closeDisplayEndpoint(); + } finally { + evaluator.close(); + } + } + } } - @Override - public void interrupt() { - evaluator.interrupt(); + private void closeDisplayEndpoint() { + if (displayEndpoint == null) { + return; + } + try { + displayEndpoint.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } - /** - * Returns notebook ClassLoader, which in the case of JavaKernel is a JShell ClassLoader. - */ @Override - protected ClassLoader getClassLoader() { - return evaluator.getClassLoader(); + public void interrupt() { + evaluator.interrupt(jShell); } /** @@ -384,31 +514,77 @@ private Builder() { @Override public JavaKernel build() { + JavaKernel[] kernelRef = new JavaKernel[1]; + UnixDomainDisplayEndpoint endpoint = null; + JShell jShell = null; + CodeEvaluator evaluator = null; + try { + endpoint = openDisplayEndpoint(kernelRef); + afterDisplayEndpointCreated.run(); + + String name = buildName(); + JupyterIO io = buildJupyterIO(buildJupyterIOEncoding()); + evaluator = buildCodeEvaluator(); + jShell = buildJShell(io, List.of("-Djjava.display.socket=" + endpoint.path())); + MagicTranspiler magicTranspiler = buildMagicTranspiler(); + + JavaKernel kernel = new JavaKernel( + name, + buildVersion(), + buildLanguageInfo(), + buildHelpLinks(), + buildHistoryManager(), + io, + buildCommManager(), + buildRenderer(), + buildMagicsResolver(magicTranspiler), + buildMagicsRegistry(), + buildExtensionsEnabled(), + buildErrorStyler(), + jShell, + evaluator + ); + kernel.displayEndpoint = endpoint; + kernelRef[0] = kernel; + return kernel; + } catch (RuntimeException | Error e) { + // Deliberately catches Error too: an OutOfMemoryError or similar during build must still release the + // JShell, display endpoint, and evaluator it has already created, and then be rethrown unchanged. + closeAfterFailure(e, jShell, endpoint, evaluator); + throw e; + } + } - String name = buildName(); - Charset jupyterEncoding = buildJupyterIOEncoding(); - CodeEvaluator evaluator = buildCodeEvaluator(name); - - JShell jShell = buildJShell(evaluator); - LanguageInfo langInfo = buildLanguageInfo(); - MagicTranspiler magicTranspiler = buildMagicTranspiler(); - - return new JavaKernel( - name, - buildVersion(), - langInfo, - buildHelpLinks(), - buildHistoryManager(), - buildJupyterIO(jupyterEncoding), - buildCommManager(), - buildRenderer(), - buildMagicsResolver(magicTranspiler), - buildMagicsRegistry(), - buildExtensionsEnabled(), - buildErrorStyler(), - jShell, - evaluator - ); + private UnixDomainDisplayEndpoint openDisplayEndpoint(JavaKernel[] kernelRef) { + try { + DisplayRequestHandler handler = displayHandler != null + ? displayHandler + : request -> applyDisplay(kernelRef[0], request); + return new UnixDomainDisplayEndpoint(displayEndpointPath, handler); + } catch (IOException e) { + throw new IllegalStateException("Unable to start display channel", e); + } + } + + private static void applyDisplay(JavaKernel kernel, DisplayRequest request) { + if (request.operation() == DisplayRequest.Operation.UPDATE) { + kernel.getIO().display.updateDisplay(request.displayId(), request.toDisplayData()); + } else { + kernel.display(request.toDisplayData()); + } + } + + private static void closeAfterFailure(Throwable failure, AutoCloseable... resources) { + for (AutoCloseable resource : resources) { + if (resource == null) { + continue; + } + try { + resource.close(); + } catch (Exception cleanup) { + if (cleanup != failure) failure.addSuppressed(cleanup); + } + } } protected List buildHelpLinks() { diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java index e52aeda..5414407 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java @@ -1,12 +1,16 @@ package org.dflib.jjava.kernel; import jdk.jshell.JShell; +import jdk.jshell.execution.JdiExecutionControlProvider; import org.dflib.jjava.jupyter.kernel.BaseKernelBuilder; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequestHandler; +import org.dflib.jjava.jupyter.kernel.JupyterIO; import org.dflib.jjava.jupyter.kernel.LanguageInfo; import org.dflib.jjava.jupyter.kernel.magic.MagicTranspiler; import org.dflib.jjava.jupyter.kernel.magic.MagicsResolver; import org.dflib.jjava.kernel.execution.CodeEvaluator; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -16,6 +20,7 @@ /** * A common builder superclass for JJavaKernel and subclasses. */ +@SuppressWarnings("unchecked") public abstract class JavaKernelBuilder< B extends JavaKernelBuilder, K extends JavaKernel> extends BaseKernelBuilder { @@ -23,9 +28,17 @@ public abstract class JavaKernelBuilder< protected long timeoutDuration; protected TimeUnit timeoutUnit; protected final List compilerOpts; + protected final List remoteVMOptions; + + // test seams: override the display endpoint address, and observe or fail the build after the endpoint exists + Path displayEndpointPath; + Runnable afterDisplayEndpointCreated = () -> { + }; + DisplayRequestHandler displayHandler; protected JavaKernelBuilder() { this.compilerOpts = new ArrayList<>(); + this.remoteVMOptions = new ArrayList<>(List.of("-classpath", System.getProperty("java.class.path"))); } public B compilerOpts(Iterable opts) { @@ -33,6 +46,26 @@ public B compilerOpts(Iterable opts) { return (B) this; } + public B remoteVMOptions(Iterable opts) { + opts.forEach(this.remoteVMOptions::add); + return (B) this; + } + + B displayEndpointPath(Path path) { + this.displayEndpointPath = path; + return (B) this; + } + + B afterDisplayEndpointCreated(Runnable hook) { + this.afterDisplayEndpointCreated = hook; + return (B) this; + } + + B displayHandler(DisplayRequestHandler handler) { + this.displayHandler = handler; + return (B) this; + } + public B timeout(long timeoutDuration, TimeUnit timeoutUnit) { this.timeoutDuration = timeoutDuration; this.timeoutUnit = Objects.requireNonNull(timeoutUnit); @@ -42,20 +75,23 @@ public B timeout(long timeoutDuration, TimeUnit timeoutUnit) { @Override public abstract K build(); - protected JShell buildJShell(CodeEvaluator evaluator) { + protected JShell buildJShell(JupyterIO io, Iterable additionalRemoteVMOptions) { + List options = new ArrayList<>(remoteVMOptions); + additionalRemoteVMOptions.forEach(options::add); return JShell.builder() - .out(System.out) - .err(System.err) - .in(System.in) - .executionEngine(evaluator.getExecControlProvider(), Map.of()) + .out(io.out) + .err(io.err) + .in(io.in) + .executionEngine(new JdiExecutionControlProvider(), Map.of()) + .remoteVMOptions(options.toArray(new String[0])) .compilerOptions(compilerOpts.toArray(new String[0])) .build(); } - protected CodeEvaluator buildCodeEvaluator(String name) { + protected CodeEvaluator buildCodeEvaluator() { long timeoutDuration = this.timeoutUnit != null ? this.timeoutDuration : -1; TimeUnit timeoutUnit = this.timeoutUnit != null ? this.timeoutUnit : TimeUnit.MILLISECONDS; - return new CodeEvaluator(name, timeoutDuration, timeoutUnit); + return new CodeEvaluator(timeoutDuration, timeoutUnit); } protected MagicsResolver buildMagicsResolver(MagicTranspiler transpiler) { diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java index e03b0ec..34db118 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java @@ -6,77 +6,55 @@ import jdk.jshell.Snippet; import jdk.jshell.SnippetEvent; import jdk.jshell.SourceCodeAnalysis; -import jdk.jshell.spi.ExecutionControl; -import jdk.jshell.spi.ExecutionControlProvider; -import jdk.jshell.spi.ExecutionEnv; import org.dflib.jjava.jupyter.kernel.BaseKernel; -import org.dflib.jjava.jupyter.telemetry.TelemetryCollector; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.List; -import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; -public class CodeEvaluator { +public class CodeEvaluator implements AutoCloseable { private static final Pattern WHITESPACE_PREFIX = Pattern.compile("(?:^|\r?\n)(?\\s*).*$"); private static final Pattern LAST_LINE = Pattern.compile("(?:^|\r?\n)(?.*)$"); - private static final String NO_MAGIC_RETURN = "\"__NO_MAGIC_RETURN\""; private static final String INDENTATION = " "; + private static final String EXECUTION_OUT_OF_MEMORY_NAME = "java.lang.OutOfMemoryError"; - private static final Method SNIPPET_CLASS_NAME_METHOD; - - static { - try { - SNIPPET_CLASS_NAME_METHOD = Snippet.class.getDeclaredMethod("classFullName"); - SNIPPET_CLASS_NAME_METHOD.setAccessible(true); - } catch (NoSuchMethodException e) { - throw new RuntimeException("Unable to access jdk.jshell.Snippet.classFullName() method.", e); - } - } - - private final String name; private final long timeoutDuration; private final TimeUnit timeoutUnit; - private final JJavaLoaderDelegate loaderDelegate; - private final JJavaExecutionControl execControl; + private final AtomicBoolean interrupted = new AtomicBoolean(); + private final ExecutorService evaluationExecutor; + private volatile ExecutorTerminationException executorFailure; - public CodeEvaluator(String name, long timeoutDuration, TimeUnit timeoutUnit) { - this.name = name; + public CodeEvaluator(long timeoutDuration, TimeUnit timeoutUnit) { this.timeoutDuration = timeoutDuration; this.timeoutUnit = timeoutUnit; - this.loaderDelegate = new JJavaLoaderDelegate(); - this.execControl = new JJavaExecutionControl(loaderDelegate, timeoutDuration, timeoutUnit); - } - - public void startThreadTelemetryCollection(TelemetryCollector collector) { - execControl.startThreadTelemetryCollection(collector); - } - - public void stopThreadTelemetryCollection() { - execControl.stopThreadTelemetryCollection(); - } - - /** - * Returns ExecutionControlProvider required by JShell. This links JShell execution engine to our code evaluator. - */ - public ExecutionControlProvider getExecControlProvider() { - return new SimpleExecControlProvider(name, execControl); + this.evaluationExecutor = Executors.newSingleThreadExecutor(r -> { + Thread thread = new Thread(r, "jjava-jshell-evaluation"); + thread.setDaemon(true); + return thread; + }); } - public Object eval(JShell shell, String code) { - + public EvaluationResult eval(JShell shell, String code) { + checkExecutorFailure(); + interrupted.set(false); + long timeoutNanos = timeoutDuration > 0 ? timeoutUnit.toNanos(timeoutDuration) : -1; + long started = System.nanoTime(); SourceCodeAnalysis sca = shell.sourceCodeAnalysis(); - Object lastResult = null; + EvaluationResult lastResult = EvaluationResult.NONE; SourceCodeAnalysis.CompletionInfo info = sca.analyzeCompletion(code); while (info.completeness().isComplete()) { - - lastResult = evalSingle(shell, info.source()); + long remainingNanos = timeoutNanos < 0 ? -1 : timeoutNanos - (System.nanoTime() - started); + lastResult = evalSingle(shell, info.source(), remainingNanos); info = sca.analyzeCompletion(info.remaining()); } @@ -87,40 +65,43 @@ public Object eval(JShell shell, String code) { return lastResult; } - protected Object evalSingle(JShell shell, String code) { - - List events = shell.eval(code.strip()); + protected EvaluationResult evalSingle(JShell shell, String code, long timeoutNanos) { + List events = evaluate(shell, code, timeoutNanos); + String result = null; - Object result = null; - - // We iterate twice to make sure throwing an early exception doesn't leak the memory - // and we `takeResult` everything. for (SnippetEvent event : events) { - if (event.status() == Snippet.Status.OVERWRITTEN) { - // if a new snippet changed some other definition, drop the older one - dropSnippet(shell, event.snippet()); + if (event.causeSnippet() != null) { continue; } - String key = event.value(); - if (key == null) { - continue; + JShellException exception = event.exception(); + if (exception != null) { + if (exception instanceof EvalException) { + EvalException evalException = (EvalException) exception; + if (EXECUTION_OUT_OF_MEMORY_NAME.equals(evalException.getExceptionClassName())) { + throw executorTerminated(shell, exception); + } + throw new RuntimeException( + evalException.getExceptionClassName() + ", " + exception.getMessage(), + exception); + } + throw new RuntimeException(exception); } - Snippet.SubKind subKind = event.snippet().subKind(); + if (event.status() != Snippet.Status.RECOVERABLE_NOT_DEFINED && !event.status().isDefined()) { + throw new CompilationException(event); + } - // Only executable snippets make their way through the machinery we have setup in the - // JJavaExecutionControl. Declarations for example simply take their default value without - // being executed. - Object value = subKind.isExecutable() - ? execControl.takeResult(key) - : event.value(); + String value = event.value(); + if (value == null) { + continue; + } - switch (subKind) { + switch (event.snippet().subKind()) { case VAR_VALUE_SUBKIND: case OTHER_EXPRESSION_SUBKIND: case TEMP_VAR_EXPRESSION_SUBKIND: - result = NO_MAGIC_RETURN.equals(value) ? null : value; + result = value; break; default: result = null; @@ -128,60 +109,65 @@ protected Object evalSingle(JShell shell, String code) { } } - for (SnippetEvent event : events) { - // If fresh snippet - if (event.causeSnippet() == null) { - JShellException e = event.exception(); - if (e != null) { - - if (e instanceof EvalException) { - EvalException ee = (EvalException) e; - switch (ee.getExceptionClassName()) { - case JJavaExecutionControl.EXECUTION_TIMEOUT_NAME: - throw new EvaluationTimeoutException(timeoutDuration, timeoutUnit, code.trim()); - case JJavaExecutionControl.EXECUTION_INTERRUPTED_NAME: - throw new EvaluationInterruptedException(code.trim()); - default: - throw new RuntimeException(ee.getExceptionClassName() + ", " + e.getMessage(), e); - } - } + return new EvaluationResult(result); + } - throw new RuntimeException(e); - } + private List evaluate(JShell shell, String code, long timeoutNanos) { + if (timeoutDuration > 0 && timeoutNanos <= 0) { + throw new EvaluationTimeoutException(timeoutDuration, timeoutUnit, code.trim()); + } - // Undefined snippets are generally bad, unless we can still recover from them. E.g., - // "Unresolved dependencies" errors are recoverable when those dependencies are defined in the later - // snippets. - if (event.status() != Snippet.Status.RECOVERABLE_NOT_DEFINED && !event.status().isDefined()) { - throw new CompilationException(event); - } + Future> evaluation = evaluationExecutor.submit(() -> shell.eval(code.strip())); + + List events; + try { + events = timeoutNanos > 0 ? evaluation.get(timeoutNanos, TimeUnit.NANOSECONDS) : evaluation.get(); + } catch (TimeoutException e) { + checkExecutorFailure(); + shell.stop(); + evaluation.cancel(true); + throw new EvaluationTimeoutException(timeoutDuration, timeoutUnit, code.trim()); + } catch (InterruptedException e) { + shell.stop(); + evaluation.cancel(true); + Thread.currentThread().interrupt(); + throw new EvaluationInterruptedException(code.trim()); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + checkExecutorFailure(); + if (interrupted.getAndSet(false)) { + throw new EvaluationInterruptedException(code.trim()); } + if (cause instanceof RuntimeException) throw (RuntimeException) cause; + if (cause instanceof Error) throw (Error) cause; + throw new RuntimeException(cause); } - return result; + if (interrupted.getAndSet(false)) { + throw new EvaluationInterruptedException(code.trim()); + } + checkExecutorFailure(); + return events; + } + + public void checkExecutorFailure() { + if (executorFailure != null) throw executorFailure; } - /** - * Try to clean up information linked to a code snippet and the snippet itself - */ - private void dropSnippet(JShell shell, Snippet snippet) { - shell.drop(snippet); - // snippet.classFullName() returns name of a wrapper class created for a snippet - String className = snippetClassName(snippet); - // check that this class is not used by other snippets - if (shell.snippets() - .map(this::snippetClassName) - .noneMatch(className::equals)) { - execControl.unloadClass(className); + public synchronized void markExecutorTerminated() { + if (executorFailure == null) { + executorFailure = new ExecutorTerminationException( + "Executor process terminated; session state is lost. Restart the kernel.", null); } } - private String snippetClassName(Snippet snippet) { - try { - return SNIPPET_CLASS_NAME_METHOD.invoke(snippet).toString(); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); + private synchronized ExecutorTerminationException executorTerminated(JShell shell, Throwable cause) { + if (executorFailure == null) { + executorFailure = new ExecutorTerminationException( + "Executor process terminated; session state is lost. Restart the kernel.", cause); + shell.close(); } + return executorFailure; } private String computeIndentation(String partialStatement) { @@ -252,32 +238,13 @@ public String isComplete(SourceCodeAnalysis sourceAnalyzer, String code) { } } - public void interrupt() { - execControl.interrupt(); - } - - public ClassLoader getClassLoader() { - return loaderDelegate.getClassLoader(); + public void interrupt(JShell shell) { + interrupted.set(true); + shell.stop(); } - static final class SimpleExecControlProvider implements ExecutionControlProvider { - - private final String name; - private final ExecutionControl control; - - public SimpleExecControlProvider(String name, ExecutionControl control) { - this.name = name; - this.control = control; - } - - @Override - public String name() { - return name; - } - - @Override - public ExecutionControl generate(ExecutionEnv env, Map parameters) { - return control; - } + @Override + public void close() { + evaluationExecutor.shutdownNow(); } } diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/EvaluationResult.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/EvaluationResult.java new file mode 100644 index 0000000..e41c181 --- /dev/null +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/EvaluationResult.java @@ -0,0 +1,10 @@ +package org.dflib.jjava.kernel.execution; + +/** + * The ordinary evaluation outcome of a cell: the JShell text of its last value or expression. A null text means the + * result is absent. It carries no Jupyter display, message, channel, or protocol types. + */ +public record EvaluationResult(String text) { + + public static final EvaluationResult NONE = new EvaluationResult(null); +} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/ExecutorTerminationException.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/ExecutorTerminationException.java new file mode 100644 index 0000000..5a44101 --- /dev/null +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/ExecutorTerminationException.java @@ -0,0 +1,7 @@ +package org.dflib.jjava.kernel.execution; + +public class ExecutorTerminationException extends RuntimeException { + public ExecutorTerminationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaExecutionControl.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaExecutionControl.java deleted file mode 100644 index f85b735..0000000 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaExecutionControl.java +++ /dev/null @@ -1,204 +0,0 @@ -package org.dflib.jjava.kernel.execution; - -import jdk.jshell.EvalException; -import jdk.jshell.execution.DirectExecutionControl; -import jdk.jshell.spi.SPIResolutionException; -import org.dflib.jjava.jupyter.telemetry.TelemetryCollector; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import java.util.concurrent.CancellationException; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * An ExecutionControl very similar to {@link jdk.jshell.execution.LocalExecutionControl} but which - * also logs the actual result of an invocation before being serialized. - */ -class JJavaExecutionControl extends DirectExecutionControl { - - // generate a semi-unique thread name prefix for each JVM run for easier detection of JJavaExecutionControl-produced threads - private static final String THREAD_NAME_PREFIX = "jjava-exec-" - + ThreadLocalRandom.current().ints(6, 'a', 'z' + 1).collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) - + "-"; - - /** - * A special "class name" for a {@link jdk.jshell.spi.ExecutionControl.UserException} such that it may be - * identified after serialization into an {@link jdk.jshell.EvalException} via {@link - * EvalException#getExceptionClassName()}. - */ - // Has spaces to not collide with a class name - public static final String EXECUTION_TIMEOUT_NAME = "Execution Timeout"; - - /** - * A special "class name" for a {@link jdk.jshell.spi.ExecutionControl.UserException} such that it may be - * identified after serialization into an {@link jdk.jshell.EvalException} via {@link - * EvalException#getExceptionClassName()} - */ - public static final String EXECUTION_INTERRUPTED_NAME = "Execution Interrupted"; - - private static final AtomicInteger EXECUTOR_THREAD_ID = new AtomicInteger(0); - - private final ExecutorService executor; - private final long timeoutDuration; - private final TimeUnit timeoutUnit; - private final Map> running; - private final Map results; - private final JJavaLoaderDelegate loaderDelegate; - private final ThreadLocal> telemetryCollector; - - public JJavaExecutionControl(JJavaLoaderDelegate loaderDelegate, long timeoutDuration, TimeUnit timeoutUnit) { - super(loaderDelegate); - - this.loaderDelegate = loaderDelegate; - this.running = new ConcurrentHashMap<>(); - this.results = new ConcurrentHashMap<>(); - - this.timeoutDuration = timeoutDuration; - this.timeoutUnit = timeoutDuration > 0 ? Objects.requireNonNull(timeoutUnit) : TimeUnit.MILLISECONDS; - this.executor = Executors.newCachedThreadPool(r -> new Thread(r, THREAD_NAME_PREFIX + EXECUTOR_THREAD_ID.getAndIncrement())); - this.telemetryCollector = new ThreadLocal<>(); - } - - @Override - public void stop() { - executor.shutdownNow(); - } - - public Object takeResult(String id) { - Object result = this.results.remove(id); - if (result == null) { - throw new IllegalStateException("No result with key: " + id); - } - - return result; - } - - public void unloadClass(String className) { - loaderDelegate.unloadClass(className); - } - - public void interrupt() { - running.forEach((id, f) -> f.cancel(true)); - } - - public void startThreadTelemetryCollection(TelemetryCollector collector) { - telemetryCollector.set(Objects.requireNonNull(collector)); - } - - public void stopThreadTelemetryCollection() { - TelemetryCollector collector = telemetryCollector.get(); - if (collector != null) { - collector.stop(); - telemetryCollector.set(null); - } - } - - /** - * This method was hijacked and actually only returns a key that can be later retrieved via - * {@link #takeResult(String)}. This should be called for every invocation as the objects are saved and not taking - * them will leak the memory. - * - * @returns the key to use for {@link #takeResult(String) looking up the result}. - */ - @Override - protected String invoke(Method doitMethod) throws Exception { - String id = UUID.randomUUID().toString(); - Object value; - - TelemetryCollector tc = threadTelemetryCollector(); - Object m = tc.measurementStart(); - try { - value = doInvoke(id, doitMethod); - } finally { - // use the same collector used for "measurementStart" call, so if the thread local is reset inside - // "doInvoke", we are still calling the matching collector - tc.measurementEnd(m); - } - - results.put(id, value); - return id; - } - - private RunException wrapInRunException(Throwable cause) { - if (cause instanceof SPIResolutionException) { - return new ResolutionException(((SPIResolutionException) cause).id(), cause.getStackTrace()); - } - - UserException ue = new UserException(String.valueOf(cause.getMessage()), cause.getClass().getName(), cause.getStackTrace()); - if (cause.getCause() != null) { - ue.initCause(wrapInRunException(cause.getCause())); - } - return ue; - } - - private Object doInvoke(String id, Method doitMethod) throws Exception { - - Future task = isNestedCall() - // run on the same thread if the invocation is done within another invocation - ? CompletableFuture.completedFuture(doitMethod.invoke(null)) - : executor.submit(() -> doitMethod.invoke(null)); - - running.put(id, task); - - try { - return timeoutDuration > 0 ? task.get(timeoutDuration, timeoutUnit) : task.get(); - } catch (CancellationException e) { - // If canceled this means that stop() or interrupt() was invoked. - if (executor.isShutdown()) { - // If the executor is shutdown, the situation is the former in which - // case the protocol is to throw an ExecutionControl.StoppedException. - throw new StoppedException(); - } else { - // The execution was purposely interrupted. - throw new UserException("Execution interrupted.", EXECUTION_INTERRUPTED_NAME, e.getStackTrace()); - } - } catch (ExecutionException e) { - - Throwable cause = e.getCause(); - if (cause instanceof InvocationTargetException) { - cause = cause.getCause(); - } - if (cause == null) { - throw new UserException("null", "Unknown Invocation Exception", e.getStackTrace()); - } else { - throw wrapInRunException(cause); - } - } catch (TimeoutException e) { - String message = String.format("Execution timed out after configured timeout of %d %s.", - timeoutDuration, - timeoutUnit.toString().toLowerCase()); - throw new UserException(message, EXECUTION_TIMEOUT_NAME, e.getStackTrace()); - } finally { - running.remove(id, task); - } - } - - private boolean isNestedCall() { - return Thread.currentThread().getName().startsWith(THREAD_NAME_PREFIX); - } - - private TelemetryCollector threadTelemetryCollector() { - TelemetryCollector tc = telemetryCollector.get(); - return tc != null ? tc : TelemetryCollector.DO_NOTHING; - } - - @Override - public String toString() { - return "JJavaExecutionControl{" + - "timeoutTime=" + timeoutDuration + - ", timeoutUnit=" + timeoutUnit + - '}'; - } -} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java deleted file mode 100644 index 8f7e39c..0000000 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java +++ /dev/null @@ -1,131 +0,0 @@ -package org.dflib.jjava.kernel.execution; - -import jdk.jshell.execution.LoaderDelegate; -import jdk.jshell.spi.ExecutionControl; -import org.dflib.jjava.jupyter.kernel.util.PathsHandler; - -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.file.Path; -import java.security.CodeSource; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -class JJavaLoaderDelegate implements LoaderDelegate { - - private static final String CLASSPATH_PROPERTY = "java.class.path"; - private static final String PATH_SEPARATOR = System.getProperty("path.separator"); - - private final Map declaredClasses; - private final Map> loadedClasses; - private final JJavaClassLoader classLoader; - - public JJavaLoaderDelegate() { - this.declaredClasses = new ConcurrentHashMap<>(); - this.loadedClasses = new ConcurrentHashMap<>(); - this.classLoader = new JJavaClassLoader(); - Thread.currentThread().setContextClassLoader(classLoader); - } - - @Override - public void load(ExecutionControl.ClassBytecodes[] cbcs) throws ExecutionControl.ClassInstallException { - boolean[] installed = new boolean[cbcs.length]; - - // Must record all defined classes before attempting to load them. Otherwise, classes depending on other, - // not yet loaded classes, may fail (see https://github.com/dflib/jjava/issues/65) - for (ExecutionControl.ClassBytecodes cbc : cbcs) { - declaredClasses.put(cbc.name(), cbc.bytecodes()); - } - - int i = 0; - for (ExecutionControl.ClassBytecodes cbc : cbcs) { - try { - Class loaderClass = classLoader.findClass(cbc.name()); - loadedClasses.put(cbc.name(), loaderClass); - } catch (ClassNotFoundException e) { - throw new ExecutionControl.ClassInstallException("Unable to load class " + cbc.name() - + ": " + e.getMessage(), installed); - } - installed[i++] = true; - } - } - - @Override - public void classesRedefined(ExecutionControl.ClassBytecodes[] cbcs) { - for (ExecutionControl.ClassBytecodes cbc : cbcs) { - declaredClasses.put(cbc.name(), cbc.bytecodes()); - } - } - - @Override - public void addToClasspath(String path) throws ExecutionControl.InternalException { - for (String next : PathsHandler.split(path)) { - try { - classLoader.addURL(Path.of(next).toUri().toURL()); - - System.setProperty( - CLASSPATH_PROPERTY, - System.getProperty(CLASSPATH_PROPERTY) + PATH_SEPARATOR + next); - - } catch (MalformedURLException e) { - throw new ExecutionControl.InternalException("Unable to resolve classpath " + next - + ": " + e.getMessage()); - } - } - } - - @Override - public Class findClass(String name) throws ClassNotFoundException { - Class klass = loadedClasses.get(name); - if (klass == null && declaredClasses.containsKey(name)) { - // check if it was removed - klass = loadClass(name); - } - if (klass == null) { - throw new ClassNotFoundException(name + " not found"); - } - return klass; - } - - private Class loadClass(String name) throws ClassNotFoundException { - return classLoader.loadClass(name); - } - - public void unloadClass(String name) { - loadedClasses.remove(name); - declaredClasses.remove(name); - } - - public ClassLoader getClassLoader() { - return classLoader; - } - - class JJavaClassLoader extends URLClassLoader { - - public JJavaClassLoader() { - super(new URL[0]); - } - - // redefine here for access from the parent class. Otherwise, the "protected" method would be inaccessible - @Override - protected void addURL(URL url) { - super.addURL(url); - } - - @Override - protected Class findClass(String name) throws ClassNotFoundException { - byte[] data = declaredClasses.get(name); - if (data == null) { - return super.findClass(name); - } - try { - return super.defineClass(name, data, 0, data.length, (CodeSource) null); - } catch (LinkageError er) { - // rethrow as ClassNotFoundException to let the caller properly handle this case - // this error could be thrown in some cases (like static method signature change) - throw new ClassNotFoundException(name, er); - } - } - } -} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/magics/TimeMagic.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/magics/TimeMagic.java index c13bf4f..3f23238 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/magics/TimeMagic.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/magics/TimeMagic.java @@ -4,13 +4,8 @@ import org.dflib.jjava.jupyter.kernel.display.DisplayData; import org.dflib.jjava.jupyter.kernel.magic.CellMagic; import org.dflib.jjava.jupyter.kernel.magic.LineMagic; -import org.dflib.jjava.jupyter.telemetry.TelemetryCollector; -import org.dflib.jjava.jupyter.telemetry.WallAndCpuTimeCollector; -import org.dflib.jjava.jupyter.telemetry.WallTimeCollector; import org.dflib.jjava.kernel.JavaKernel; -import java.lang.management.ManagementFactory; -import java.lang.management.ThreadMXBean; import java.util.List; import java.util.UUID; @@ -32,68 +27,19 @@ public DisplayData eval(JavaKernel kernel, List args, String body) throw } private DisplayData timeAndRunCode(JavaKernel kernel, String code) { - ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); - - TelemetryCollector collector = WallAndCpuTimeCollector.canMeasureCpuTimes(threadMXBean) - ? new TMWallAndCpuTimeCollector(kernel, threadMXBean) - : new TMWallTimeCollector(kernel); - - kernel.getEvaluator().startThreadTelemetryCollection(collector); - + long started = System.nanoTime(); try { return kernel.evalBuilder(code) .resolveMagics() .renderResults() .eval(); } finally { - kernel.getEvaluator().stopThreadTelemetryCollection(); + displayWallTime(kernel, System.nanoTime() - started); } } - private static void displayCpuTime(BaseKernel kernel, long userTimeNanos, long sysTimeNanos, long totalTimeNanos) { - String cpuTimes = String.format("CPU times: user %.3f s, sys %.3f s, total %.3f s", - userTimeNanos / NANONS_IN_SEC, - sysTimeNanos / NANONS_IN_SEC, - totalTimeNanos / NANONS_IN_SEC); - - kernel.display(new DisplayData(cpuTimes).setDisplayId(UUID.randomUUID().toString())); - } - private static void displayWallTime(BaseKernel kernel, long wallTimeNanos) { String wallTime = String.format("Wall time: %.3f s", wallTimeNanos / NANONS_IN_SEC); kernel.display(new DisplayData(wallTime).setDisplayId(UUID.randomUUID().toString())); } - - static class TMWallTimeCollector extends WallTimeCollector { - private final BaseKernel kernel; - - public TMWallTimeCollector(BaseKernel kernel) { - this.kernel = kernel; - } - - @Override - protected void onStop(long wallTimeNanos) { - displayWallTime(kernel, wallTimeNanos); - } - } - - - static class TMWallAndCpuTimeCollector extends WallAndCpuTimeCollector { - private final BaseKernel kernel; - - public TMWallAndCpuTimeCollector(BaseKernel kernel, ThreadMXBean threadMXBean) { - super(threadMXBean); - this.kernel = kernel; - } - - @Override - protected void onStop(long wallTimeNanos, long userTimeNanos, long totalTimeNanos) { - // sanity check: CPU and user times can't be measured sometimes - if (userTimeNanos != -1 && totalTimeNanos != -1) { - displayCpuTime(kernel, userTimeNanos, totalTimeNanos - userTimeNanos, totalTimeNanos); - } - - displayWallTime(kernel, wallTimeNanos); - } - } } diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/jupyter/kernel/CapturingShellEnvironment.java b/jjava-kernel/src/test/java/org/dflib/jjava/jupyter/kernel/CapturingShellEnvironment.java new file mode 100644 index 0000000..9921e92 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/jupyter/kernel/CapturingShellEnvironment.java @@ -0,0 +1,32 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.dflib.jjava.jupyter.channels.ShellReplyEnvironment; +import org.dflib.jjava.jupyter.messages.Message; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Test helper: records every message a kernel publishes. Lives in the kernel package so it can reach the protected + * {@code setEnv} of {@link JupyterIO}. + */ +public class CapturingShellEnvironment extends ShellReplyEnvironment { + + public final List> published = Collections.synchronizedList(new ArrayList<>()); + + private CapturingShellEnvironment() { + super(null, null, null, null); + } + + public static CapturingShellEnvironment attach(JupyterIO io) { + CapturingShellEnvironment env = new CapturingShellEnvironment(); + io.setEnv(env); + return env; + } + + @Override + public void publish(Message message) { + published.add(message); + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelDisplayFailureTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelDisplayFailureTest.java new file mode 100644 index 0000000..3d61adb --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelDisplayFailureTest.java @@ -0,0 +1,77 @@ +package org.dflib.jjava.kernel; + +import org.dflib.jjava.jupyter.kernel.CapturingShellEnvironment; +import org.dflib.jjava.jupyter.messages.publish.PublishDisplayData; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Non-portable payloads are rejected in the executor before any delivery: nothing is published, and the next cell + * still evaluates. The stalled-peer case is not covered yet (needs a blocking endpoint handler seam). + */ +class JavaKernelDisplayFailureTest { + + private static final String IMPORT = "import static org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics.*;\n"; + + @Test + void nonPortablePayloadsAreRejectedWithoutPublishingAndKernelContinues() { + JavaKernel kernel = JavaKernel.builder().extensionsEnabled(false).build(); + try { + CapturingShellEnvironment env = CapturingShellEnvironment.attach(kernel.getIO()); + + // a cyclic list and a non-finite double, both built in the child + // Rejected by renderPortable before the protocol check: a raw collection is not a supported display value. + assertFailsWithMessage(kernel, + "java.util.List cyclic = new java.util.ArrayList<>(); cyclic.add(cyclic); display(cyclic);", + "Non-portable display value"); + assertFailsWithMessage(kernel, + "display(java.util.List.of(Double.NaN));", + "Non-portable display value"); + + assertTrue(env.published.stream().noneMatch(m -> m.getContent() instanceof PublishDisplayData), + "a rejected payload was published"); + + assertEquals("42", kernel.evalBuilder("40 + 2").resolveMagics().eval()); + } finally { + kernel.onShutdown(false); + } + } + + // Pinned current behavior: a bare scalar is rendered to text before the protocol's non-finite check, which applies to containers. + @Test + void bareNonFiniteScalarIsPublishedAsTextNaN() { + JavaKernel kernel = JavaKernel.builder().extensionsEnabled(false).build(); + try { + CapturingShellEnvironment env = CapturingShellEnvironment.attach(kernel.getIO()); + + kernel.evalBuilder(IMPORT + "display(Double.NaN);").resolveMagics().eval(); + + PublishDisplayData published = assertInstanceOf(PublishDisplayData.class, env.published.get(0).getContent()); + assertEquals("NaN", published.getData().get("text/plain")); + } finally { + kernel.onShutdown(false); + } + } + + private static void assertFailsWithMessage(JavaKernel kernel, String code, String fragment) { + var evaluation = kernel.evalBuilder(IMPORT + code).resolveMagics(); + RuntimeException failure = org.junit.jupiter.api.Assertions.assertThrows( + RuntimeException.class, + evaluation::eval); + assertTrue(String.valueOf(failure.getMessage()).contains(fragment) + || causeChainContains(failure, fragment), + "expected message containing '" + fragment + "' but was: " + failure); + } + + private static boolean causeChainContains(Throwable t, String fragment) { + for (Throwable c = t; c != null; c = c.getCause()) { + if (String.valueOf(c.getMessage()).contains(fragment)) { + return true; + } + } + return false; + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEndpointLifecycleTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEndpointLifecycleTest.java new file mode 100644 index 0000000..1bb90e2 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEndpointLifecycleTest.java @@ -0,0 +1,43 @@ +package org.dflib.jjava.kernel; + +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class JavaKernelEndpointLifecycleTest { + + @Test + void endpointSocketIsRemovedOnShutdown() throws Exception { + Path socket = Files.createTempDirectory("uds-lifecycle").resolve("display.sock"); + + JavaKernel kernel = JavaKernel.builder().extensionsEnabled(false).displayEndpointPath(socket).build(); + assertTrue(Files.exists(socket)); + + kernel.onShutdown(false); + assertFalse(Files.exists(socket)); + } + + @Test + void buildFailureAfterEndpointCreationRemovesEndpoint() throws Exception { + Path socket = Files.createTempDirectory("uds-lifecycle").resolve("display.sock"); + AtomicBoolean hookRan = new AtomicBoolean(); + + assertThrows(IllegalStateException.class, () -> JavaKernel.builder() + .extensionsEnabled(false) + .displayEndpointPath(socket) + .afterDisplayEndpointCreated(() -> { + hookRan.set(true); + throw new IllegalStateException("boom"); + }) + .build()); + + assertTrue(hookRan.get()); + assertFalse(Files.exists(socket)); + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEvalBuilderTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEvalBuilderTest.java new file mode 100644 index 0000000..5446776 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEvalBuilderTest.java @@ -0,0 +1,86 @@ +package org.dflib.jjava.kernel; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.fail; + +public class JavaKernelEvalBuilderTest { + + private JavaKernel kernel; + + @BeforeEach + void setUp() { + kernel = JavaKernel.builder().extensionsEnabled(false).build(); + } + + @AfterEach + void tearDown() { + kernel.onShutdown(false); + } + + @Test + void rendersManagerLineAndCellMagicsRegardlessOfBuilderOrder() { + AtomicInteger calls = new AtomicInteger(); + kernel.getMagicsRegistry().registerLineMagic("probe", (k, args) -> { + assertSame(kernel, k); + assertEquals(List.of("first", "second"), args); + return "magic-" + calls.incrementAndGet(); + }); + kernel.getMagicsRegistry().registerCellMagic("probeCell", (k, args, body) -> { + assertSame(kernel, k); + assertEquals(List.of("first", "second"), args); + assertEquals("cell body", body); + return "cell-result"; + }); + + DisplayData renderFirst = kernel.evalBuilder("%probe first second").renderResults().resolveMagics().eval(); + DisplayData resolveFirst = kernel.evalBuilder("%probe first second") + .resolveMagics() + .renderResults() + .renderResults() + .eval(); + DisplayData cellResult = kernel.evalBuilder("%%probeCell first second\ncell body") + .renderResults() + .resolveMagics() + .eval(); + + assertEquals("magic-1", renderFirst.getData().get("text/plain")); + assertEquals("magic-2", resolveFirst.getData().get("text/plain")); + assertEquals("cell-result", cellResult.getData().get("text/plain")); + } + + @Test + void preservesNullAndDisplayDataMagicResults() { + kernel.getMagicsRegistry().registerLineMagic("nothing", (k, args) -> null); + assertNull(kernel.evalBuilder("%nothing").renderResults().resolveMagics().eval()); + + DisplayData returned = new DisplayData("already rendered"); + kernel.getMagicsRegistry().registerLineMagic("bundle", (k, args) -> returned); + assertSame(returned, kernel.evalBuilder("%bundle").resolveMagics().renderResults().eval()); + } + + @Test + void textualNullResultIsPresentWhileAbsentResultIsNot() { + DisplayData textualNull = kernel.evalBuilder("String nothing = null;\nnothing").renderResults().eval(); + assertEquals("null", textualNull.getData().get("text/plain")); + assertNull(kernel.evalBuilder("int declared = 1;").renderResults().eval()); + } + + @Test + void doesNotUseManagerRendererForExecutorResults() { + kernel.getRenderer().createRegistration(String.class) + .register((value, context) -> fail("Executor result must not be rendered in the manager")); + + DisplayData executorResult = kernel.evalBuilder("40 + 2").renderResults().eval(); + assertEquals("42", executorResult.getData().get("text/plain")); + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelExtensionsLifecycleTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelExtensionsLifecycleTest.java index bed24db..4d1b067 100644 --- a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelExtensionsLifecycleTest.java +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelExtensionsLifecycleTest.java @@ -22,7 +22,9 @@ public void extraClasspathExtension() throws Exception { String extraClasspath = PathsHandler.joinPaths(List.of(extensionJar)); String extInstalledProp = "ext.installs:org.dflib.jjava.kernel.test.ExtraClasspathExtension"; + String extUninstalledProp = "ext.uninstalls:org.dflib.jjava.kernel.test.ExtraClasspathExtension"; System.clearProperty(extInstalledProp); + System.clearProperty(extUninstalledProp); JavaKernel kernel = JavaKernel .builder() @@ -42,6 +44,8 @@ public void extraClasspathExtension() throws Exception { } assertNull(System.getProperty(extInstalledProp)); + assertEquals("1", System.getProperty(extUninstalledProp)); + System.clearProperty(extUninstalledProp); } @Test @@ -63,10 +67,12 @@ public void evalExtension() throws Exception { kernel.addToClasspath(extraClasspath); Object installed = kernel.evalBuilder("evalExtensionInstalled").eval(); - assertEquals(true, installed, "EvalExtension should have been installed"); + assertEquals("true", installed, "EvalExtension should have been installed"); + + assertEquals("manager:argument", kernel.evalBuilder("%testManagerMagic argument").resolveMagics().eval()); Object result = kernel.evalBuilder("evalValue").eval(); - assertEquals("Test message", result.toString(), "eval() call was not successful"); + assertEquals("\"Test message\"", result.toString(), "eval() call was not successful"); } finally { kernel.onShutdown(false); } @@ -95,10 +101,10 @@ public void libraryExtension() throws Exception { kernel.addToClasspath(extraClasspath); Object installed = kernel.evalBuilder("externalLibraryExtensionInstalled").eval(); - assertEquals(true, installed, "ExternalLibraryExtension should have been installed"); + assertEquals("true", installed, "ExternalLibraryExtension should have been installed"); Object result = kernel.evalBuilder("externalLibraryValue").eval(); - assertEquals("Test message", result.toString(), "Library class method call was not successful"); + assertEquals("\"Test message\"", result.toString(), "Library class method call was not successful"); } finally { kernel.onShutdown(false); } diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelStalledDisplayTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelStalledDisplayTest.java new file mode 100644 index 0000000..04ddf7e --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelStalledDisplayTest.java @@ -0,0 +1,62 @@ +package org.dflib.jjava.kernel; + +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * A manager that accepts a display request but never answers. The executor must fail within the response bound and + * report the outcome as uncertain (never retried), and the kernel must still evaluate afterwards. + */ +class JavaKernelStalledDisplayTest { + + private static final String IMPORT = "import static org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics.*;\n"; + // executor RESPONSE bound is 10 s; allow a scheduling tolerance of 5 s + private static final long BOUND_MILLIS = 15_000; + + @Test + void stalledManagerFailsUncertainWithinBoundAndKernelStaysUsable() { + CountDownLatch release = new CountDownLatch(1); + JavaKernel kernel = JavaKernel.builder() + .extensionsEnabled(false) + .displayHandler(request -> awaitQuietly(release)) + .build(); + try { + var stalledDisplay = kernel.evalBuilder(IMPORT + "display(\"stalled\")").resolveMagics(); + long start = System.nanoTime(); + RuntimeException failure = assertThrows(RuntimeException.class, stalledDisplay::eval); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + + assertTrue(elapsedMillis < BOUND_MILLIS, "took " + elapsedMillis + " ms"); + assertTrue(chainContains(failure, "Failed to send display data"), String.valueOf(failure)); + + release.countDown(); + assertEquals("42", kernel.evalBuilder("40 + 2").resolveMagics().eval()); + } finally { + release.countDown(); + kernel.onShutdown(false); + } + } + + private static void awaitQuietly(CountDownLatch latch) { + try { + latch.await(60, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private static boolean chainContains(Throwable t, String fragment) { + for (Throwable c = t; c != null; c = c.getCause()) { + if (String.valueOf(c.getMessage()).contains(fragment)) { + return true; + } + } + return false; + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaNotebookStaticsRoutingTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaNotebookStaticsRoutingTest.java new file mode 100644 index 0000000..a2ede71 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaNotebookStaticsRoutingTest.java @@ -0,0 +1,28 @@ +package org.dflib.jjava.kernel; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class JavaNotebookStaticsRoutingTest { + + @Test + void executorKernelAlwaysFailsWithoutReadingKernel() { + assertEquals( + "JavaKernel.kernel() cannot expose the managing JVM from executor code", + assertThrows(UnsupportedOperationException.class, ExecutorJavaNotebookStatics::kernel).getMessage()); + } + + @SuppressWarnings("removal") + @Test + void executorGetKernelInstanceAlwaysFails() { + assertThrows(UnsupportedOperationException.class, ExecutorJavaNotebookStatics::getKernelInstance); + } + + @Test + void managerKernelWithoutRunningKernelFailsWithNullPointerFromKernelAccess() { + // Accepted outcome (spec Story 4 #8): no lifecycle message; the kernel access itself fails. + assertThrows(NullPointerException.class, JavaNotebookStatics::kernel); + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/OutputBoundaryDependencyTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/OutputBoundaryDependencyTest.java new file mode 100644 index 0000000..968ef74 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/OutputBoundaryDependencyTest.java @@ -0,0 +1,65 @@ +package org.dflib.jjava.kernel; + +import jdk.jshell.JShell; +import org.dflib.jjava.kernel.execution.CodeEvaluator; +import org.dflib.jjava.kernel.execution.EvaluationResult; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.spi.ToolProvider; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class OutputBoundaryDependencyTest { + + @Test + public void evaluatorAndResultModelDoNotDependOnJupyterPresentation() { + List violations = new ArrayList<>(); + List edges = edges(Path.of("target", "classes")); + assertFalse(edges.isEmpty(), "jdeps produced no edges; the check would pass vacuously"); + for (String[] edge : edges) { + String from = edge[0]; + String to = edge[1]; + boolean evaluator = from.startsWith(CodeEvaluator.class.getName()); + boolean result = from.startsWith(EvaluationResult.class.getName()); + if ((evaluator || result) && (to.startsWith("org.dflib.jjava.jupyter.kernel.display.") + || to.startsWith("org.dflib.jjava.jupyter.messages.") + || to.startsWith("org.dflib.jjava.jupyter.channels."))) { + violations.add(from + " -> " + to); + } + } + assertTrue(violations.isEmpty(), "forbidden edges: " + violations); + } + + @Test + public void evalReturnsEvaluationResult() throws Exception { + assertEquals(EvaluationResult.class, + CodeEvaluator.class.getMethod("eval", JShell.class, String.class).getReturnType()); + } + + private static List edges(Path classes) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ToolProvider.findFirst("jdeps").orElseThrow().run( + new PrintStream(out, true, StandardCharsets.UTF_8), System.err, + "-verbose:class", classes.toString()); + + List edges = new ArrayList<>(); + for (String line : out.toString(StandardCharsets.UTF_8).split("\n")) { + int arrow = line.indexOf(" -> "); + if (arrow < 0) { + continue; + } + edges.add(new String[]{ + line.substring(0, arrow).trim(), + line.substring(arrow + 4).trim().split("\\s+")[0]}); + } + return edges; + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/RemoteDisplayRoundTripTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/RemoteDisplayRoundTripTest.java new file mode 100644 index 0000000..497f9d6 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/RemoteDisplayRoundTripTest.java @@ -0,0 +1,41 @@ +package org.dflib.jjava.kernel; + +import org.dflib.jjava.jupyter.kernel.CapturingShellEnvironment; +import org.dflib.jjava.jupyter.messages.publish.PublishDisplayData; +import org.dflib.jjava.jupyter.messages.publish.PublishUpdateDisplayData; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +/** + * A display and an update made by user code in the executor JVM are published by the manager. + */ +public class RemoteDisplayRoundTripTest { + + private static final String IMPORT = "import static org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics.*;\n"; + + @Test + public void executorDisplayAndUpdatePublishOnManager() { + JavaKernel kernel = JavaKernel.builder().extensionsEnabled(false).build(); + try { + CapturingShellEnvironment env = CapturingShellEnvironment.attach(kernel.getIO()); + + // eval() returns the JShell text of the String result, so the id arrives with its quotes + String id = ((String) kernel.evalBuilder(IMPORT + "display(\"hello\")").resolveMagics().eval()).replace("\"", ""); + + PublishDisplayData display = assertInstanceOf(PublishDisplayData.class, env.published.get(0).getContent()); + assertEquals(id, display.getDisplayId()); + assertEquals("hello", display.getData().get("text/plain")); + + kernel.evalBuilder(IMPORT + "updateDisplay(\"" + id + "\", \"updated\")").resolveMagics().eval(); + + PublishUpdateDisplayData update = assertInstanceOf( + PublishUpdateDisplayData.class, + env.published.get(1).getContent()); + assertEquals(id, update.getDisplayId()); + } finally { + kernel.onShutdown(false); + } + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/StandaloneMagicDispatchTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/StandaloneMagicDispatchTest.java new file mode 100644 index 0000000..2e850fe --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/StandaloneMagicDispatchTest.java @@ -0,0 +1,54 @@ +package org.dflib.jjava.kernel; + +import org.dflib.jjava.jupyter.kernel.BaseKernel; +import org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics; +import org.dflib.jjava.jupyter.kernel.magic.CellMagic; +import org.dflib.jjava.jupyter.kernel.magic.LineMagic; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Standalone magics are dispatched in the manager. If the generated call had been sent to the executor, the executor + * helper would throw UnsupportedOperationException, so a normal result also shows the executor never received it. + * Inline magics are the opposite: they resolve to an executor helper call. + */ +class StandaloneMagicDispatchTest { + + @Test + void standaloneLineAndCellMagicsDispatchInTheManager() { + List seen = new ArrayList<>(); + LineMagic line = (kernel, args) -> { + seen.add(kernel); + return "line"; + }; + CellMagic cell = (kernel, args, body) -> { + seen.add(kernel); + return body; + }; + + JavaKernel kernel = JavaKernel.builder() + .extensionsEnabled(false) + .lineMagic("echoLine", line) + .cellMagic("echoCell", cell) + .build(); + try { + assertEquals("line", kernel.evalBuilder("%echoLine hi").resolveMagics().eval()); + assertEquals("body", kernel.evalBuilder("%%echoCell x\nbody").resolveMagics().eval()); + + assertEquals(2, seen.size()); + assertSame(kernel, seen.get(0)); + assertSame(kernel, seen.get(1)); + + String inline = kernel.getMagicsResolver().resolve("x = %echoLine hi"); + assertTrue(inline.contains(ExecutorNotebookStatics.class.getName() + ".lineMagic("), inline); + } finally { + kernel.onShutdown(false); + } + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/CodeEvaluatorTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/CodeEvaluatorTest.java new file mode 100644 index 0000000..a007380 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/CodeEvaluatorTest.java @@ -0,0 +1,55 @@ +package org.dflib.jjava.kernel.execution; + +import jdk.jshell.EvalException; +import jdk.jshell.JShell; +import jdk.jshell.execution.JdiExecutionControlProvider; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class CodeEvaluatorTest { + + @Test + public void evaluatesLastExpressionAndLeavesDeclarationsWithoutResults() { + try (JShell shell = newShell(); CodeEvaluator evaluator = new CodeEvaluator(-1, TimeUnit.SECONDS)) { + assertEquals("42", evaluator.eval(shell, "int base = 40;\nbase + 2").text()); + assertEquals(EvaluationResult.NONE, evaluator.eval(shell, "int declared = 1;")); + assertEquals("\"__NO_MAGIC_RETURN\"", evaluator.eval(shell, "\"__NO_MAGIC_RETURN\"").text()); + } + } + + @Test + public void userExceptionDoesNotPoisonSubsequentEvaluation() { + try (JShell shell = newShell(); CodeEvaluator evaluator = new CodeEvaluator(-1, TimeUnit.SECONDS)) { + RuntimeException failure = assertThrows( + RuntimeException.class, + () -> evaluator.eval(shell, "throw new IllegalStateException(\"expected\");")); + + assertInstanceOf(EvalException.class, failure.getCause()); + assertEquals("42", evaluator.eval(shell, "40 + 2").text()); + } + } + + @Test + public void timeoutDoesNotPoisonSubsequentEvaluation() { + try (JShell shell = newShell(); CodeEvaluator evaluator = new CodeEvaluator(5, TimeUnit.SECONDS)) { + assertEquals("2", evaluator.eval(shell, "1 + 1").text()); + assertThrows( + EvaluationTimeoutException.class, + () -> evaluator.eval(shell, "try { Thread.sleep(30000L); } catch (InterruptedException ignored) {}")); + + assertEquals("42", evaluator.eval(shell, "40 + 2").text()); + } + } + + private JShell newShell() { + return JShell.builder() + .executionEngine(new JdiExecutionControlProvider(), Map.of()) + .build(); + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegateTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegateTest.java deleted file mode 100644 index 0e0f86e..0000000 --- a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegateTest.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.dflib.jjava.kernel.execution; - -import jdk.jshell.spi.ExecutionControl; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.net.URISyntaxException; - -import static org.junit.jupiter.api.Assertions.*; - -public class JJavaLoaderDelegateTest { - - @Test - public void addToClasspath() throws URISyntaxException, ExecutionControl.InternalException { - File extraCp = new File(getClass().getResource("cp").toURI()); - assertTrue(extraCp.isDirectory()); - - JJavaLoaderDelegate ld = new JJavaLoaderDelegate(); - - String cp1 = System.getProperty("java.class.path"); - - ld.addToClasspath(extraCp.getAbsolutePath()); - String cp2 = System.getProperty("java.class.path"); - assertEquals(cp1 + System.getProperty("path.separator") + extraCp.getAbsolutePath(), cp2); - - // TODO: test that classes from this location were actually loaded - } -} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlRedefinitionTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlRedefinitionTest.java new file mode 100644 index 0000000..c3ce56c --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlRedefinitionTest.java @@ -0,0 +1,34 @@ +package org.dflib.jjava.kernel.execution; + +import org.dflib.jjava.kernel.JavaKernel; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class JdiExecutionControlRedefinitionTest { + + @Test + public void classAndMethodRedefinitionKeepsSavedValues() { + JavaKernel kernel = JavaKernel.builder().extensionsEnabled(false).build(); + try { + kernel.evalBuilder("class Stateful { int value() { return 1; } }").eval(); + kernel.evalBuilder("Stateful instance = new Stateful();").eval(); + kernel.evalBuilder("int saved = instance.value();").eval(); + assertEquals("1", kernel.evalBuilder("saved").eval()); + + kernel.evalBuilder("class Stateful { int value() { return 2; } }").eval(); + + assertEquals("1", kernel.evalBuilder("saved").eval()); + assertEquals("2", kernel.evalBuilder("instance.value()").eval()); + assertEquals("2", kernel.evalBuilder("new Stateful().value()").eval()); + + kernel.evalBuilder("int currentValue() { return 10; }").eval(); + kernel.evalBuilder("int savedMethodValue = currentValue();").eval(); + kernel.evalBuilder("int currentValue() { return 20; }").eval(); + assertEquals("10", kernel.evalBuilder("savedMethodValue").eval()); + assertEquals("20", kernel.evalBuilder("currentValue()").eval()); + } finally { + kernel.onShutdown(false); + } + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlTest.java new file mode 100644 index 0000000..b85f9b5 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlTest.java @@ -0,0 +1,32 @@ +package org.dflib.jjava.kernel.execution; + +import org.dflib.jjava.kernel.JavaKernel; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +public class JdiExecutionControlTest { + + @Test + public void usesRemoteExecutorForPersistentStateImportsAndOptions() { + JavaKernel kernel = JavaKernel.builder() + .extensionsEnabled(false) + .remoteVMOptions(List.of("-Djjava.test.executor=remote")) + .build(); + try { + long executorPid = Long.parseLong(kernel.evalBuilder("ProcessHandle.current().pid()").eval()); + assertNotEquals(ProcessHandle.current().pid(), executorPid); + + kernel.evalBuilder("int saved = 40;").eval(); + kernel.evalBuilder("import java.time.LocalDate;").eval(); + assertEquals("42", kernel.evalBuilder("saved + 2").eval()); + assertEquals(Long.toString(executorPid), kernel.evalBuilder("ProcessHandle.current().pid()").eval()); + assertEquals("\"remote\"", kernel.evalBuilder("System.getProperty(\"jjava.test.executor\")").eval()); + } finally { + kernel.onShutdown(false); + } + } +} diff --git a/jjava-kernel/src/test/resources/extensions/classpath/org/dflib/jjava/kernel/test/ExtraClasspathExtension.java b/jjava-kernel/src/test/resources/extensions/classpath/org/dflib/jjava/kernel/test/ExtraClasspathExtension.java index 0fc927c..57e276d 100644 --- a/jjava-kernel/src/test/resources/extensions/classpath/org/dflib/jjava/kernel/test/ExtraClasspathExtension.java +++ b/jjava-kernel/src/test/resources/extensions/classpath/org/dflib/jjava/kernel/test/ExtraClasspathExtension.java @@ -21,5 +21,8 @@ public void install(BaseKernel kernel) { public void uninstall(BaseKernel kernel) { String key = "ext.installs:" + getClass().getName(); System.clearProperty(key); + String uninstallKey = "ext.uninstalls:" + getClass().getName(); + int count = Integer.parseInt(System.getProperty(uninstallKey, "0")); + System.setProperty(uninstallKey, String.valueOf(count + 1)); } } diff --git a/jjava-kernel/src/test/resources/extensions/eval/org/dflib/jjava/kernel/test/EvalExtension.java b/jjava-kernel/src/test/resources/extensions/eval/org/dflib/jjava/kernel/test/EvalExtension.java index 36f71a8..afcbcf8 100644 --- a/jjava-kernel/src/test/resources/extensions/eval/org/dflib/jjava/kernel/test/EvalExtension.java +++ b/jjava-kernel/src/test/resources/extensions/eval/org/dflib/jjava/kernel/test/EvalExtension.java @@ -2,6 +2,7 @@ import org.dflib.jjava.jupyter.Extension; import org.dflib.jjava.jupyter.kernel.BaseKernel; +import org.dflib.jjava.jupyter.kernel.magic.LineMagic; public class EvalExtension implements Extension { @@ -9,5 +10,7 @@ public class EvalExtension implements Extension { public void install(BaseKernel kernel) { kernel.evalBuilder("var evalValue = \"Test message\";").eval(); kernel.evalBuilder("var evalExtensionInstalled = true;").eval(); + LineMagic probe = (k, args) -> "manager:" + args.get(0); + kernel.getMagicsRegistry().registerLineMagic("testManagerMagic", probe); } } diff --git a/jjava-launcher/pom.xml b/jjava-launcher/pom.xml deleted file mode 100644 index 90c72c6..0000000 --- a/jjava-launcher/pom.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - 4.0.0 - - - org.dflib.jjava - jjava-parent - 1.0-SNAPSHOT - - - jjava-launcher - Java Jupyter Kernel launcher - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - org.dflib.jjava.launcher.KernelLauncher - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-artifacts - package - - copy - - - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - ../kernelspec/java - jjava-launcher.jar - - - - - - - - org.apache.maven.plugins - maven-clean-plugin - - - - ../kernelspec/java - - jjava-launcher.jar - - - - - - - - - diff --git a/jjava-launcher/src/main/java/org/dflib/jjava/launcher/KernelLauncher.java b/jjava-launcher/src/main/java/org/dflib/jjava/launcher/KernelLauncher.java deleted file mode 100644 index 348d91d..0000000 --- a/jjava-launcher/src/main/java/org/dflib/jjava/launcher/KernelLauncher.java +++ /dev/null @@ -1,116 +0,0 @@ -package org.dflib.jjava.launcher; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.logging.Logger; - -/** - * The launcher responsible for creating a new process to run the JJava kernel. - *

- * The command to run the kernel is constructed by combining the JVM executable, - * the JJAVA_JVM_OPTS environment variable (if set), and the kernel arguments. - */ -// TODO: stop using exceptions for flow control. We are dealing with a single class and a main method here. -// All the errors should be reported as exit code and messages printed to stderr -public class KernelLauncher { - - // Since "launcher" is kind of a part of the "distro", we are using JUL directly, - // unlike the rest of the framework that uses SLF4J, and is only linked to JUL in the "distro" - private static final Logger LOGGER = Logger.getLogger("KernelLauncher"); - - // More JUL stuff... This is copied verbatim from JJava class to mimic Jupyter log format - // TODO: does it make sense to pass this property to the child JJava process via -D, - // to avoid duplicating this logic? - private static final String JUL_JUPYTER_LOG_FORMAT = "[%4$.1s %1$tF %1$tT.%1$tL %3$s] %5$s%n"; - - private static final String JJAVA_JVM_OPTS = "JJAVA_JVM_OPTS"; - - private final List args; - - public KernelLauncher(List args) { - this.args = args; - } - - public static void main(String[] args) { - System.setProperty("java.util.logging.SimpleFormatter.format", JUL_JUPYTER_LOG_FORMAT); - KernelLauncher launcher = new KernelLauncher(Arrays.asList(args)); - int exitCode = launcher.launchKernel(); - System.exit(exitCode); - } - - /** - * Launches a new process to run the JJava kernel with the given arguments. - *

- * The kernel inherits the standard input, output and error streams from the parent process. - * The method blocks until the kernel terminates. - * It also provides shutdown hooks to terminate the kernel when the JVM terminates. - * - * @return the exit code of the kernel - */ - public int launchKernel() { - List command = buildCommand(args); - LOGGER.info(() -> "Starting kernel: " + command); - try { - ProcessBuilder pb = new ProcessBuilder(command); - pb.inheritIO(); - Process kernel = pb.start(); - addShutdownHook(kernel); - return kernel.waitFor(); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - private void addShutdownHook(Process kernel) { - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - LOGGER.fine("Terminating kernel..."); - kernel.destroy(); - try { - kernel.waitFor(); - LOGGER.info("Kernel terminated."); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - })); - } - - private List buildCommand(List args) { - if (args.size() < 2) { - throw new RuntimeException(buildErrorMessage(args)); - } - String kernelPath = args.get(0); - String connectionFile = args.get(1); - - List command = new ArrayList<>(); - command.add("java"); - - // Get JVM options from environment variable if provided - String jvmOptions = System.getenv(JJAVA_JVM_OPTS); - if (jvmOptions != null && !jvmOptions.isBlank()) { - String[] options = jvmOptions.split("\\s+"); - command.addAll(Arrays.asList(options)); - } - - // Add JVM option for JShell permissions - command.add("--add-opens"); - command.add("jdk.jshell/jdk.jshell=ALL-UNNAMED"); - command.add("-jar"); - command.add(kernelPath); - command.add(connectionFile); - return command; - } - - private String buildErrorMessage(List args) { - - switch (args.size()) { - case 0: - return "Missing arguments: %n"; - case 1: - return String.format("Missing arguments: %s %n", args.get(0)); - default: - return String.format("Arguments provided: %s%n", String.join(", ", args)); - } - } -} diff --git a/kernelspec/java/kernel.json b/kernelspec/java/kernel.json index 9ef084a..f5374ed 100644 --- a/kernelspec/java/kernel.json +++ b/kernelspec/java/kernel.json @@ -2,7 +2,6 @@ "argv": [ "java", "-jar", - "{resource_dir}/jjava-launcher.jar", "{resource_dir}/jjava.jar", "{connection_file}" ], diff --git a/pom.xml b/pom.xml index b4e10a4..78cae47 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,6 @@ jjava-jupyter jjava-kernel jjava-maven - jjava-launcher jjava-distro @@ -44,7 +43,7 @@ UTF-8 UTF-8 - 11 + 21 3.9.9 2.4.0