From 7d3d3586e3dfe352572f705b789dd553d3aee785 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Thu, 12 Mar 2026 10:41:20 +0100 Subject: [PATCH 01/19] Working on a breaking test script --- .vscode/launch.json | 7 ++ src/org/rascalmpl/compiler/BreakTest.java | 87 +++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 src/org/rascalmpl/compiler/BreakTest.java diff --git a/.vscode/launch.json b/.vscode/launch.json index fa9529bef36..da94c795cbb 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,13 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "type": "java", + "name": "BreakTest", + "request": "launch", + "mainClass": "org.rascalmpl.compiler.BreakTest", + "projectName": "rascal" + }, { "type": "java", "name": "RascalCheck", diff --git a/src/org/rascalmpl/compiler/BreakTest.java b/src/org/rascalmpl/compiler/BreakTest.java new file mode 100644 index 00000000000..5a2ade2ac83 --- /dev/null +++ b/src/org/rascalmpl/compiler/BreakTest.java @@ -0,0 +1,87 @@ +package org.rascalmpl.compiler; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Reader; +import java.io.StringWriter; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.rascalmpl.debug.IRascalMonitor; +import org.rascalmpl.interpreter.ITestResultListener; +import org.rascalmpl.interpreter.TestEvaluator; +import org.rascalmpl.shell.RascalShell; +import org.rascalmpl.shell.ShellEvaluatorFactory; +import org.rascalmpl.test.infrastructure.RascalJUnitTestRunner; + +import io.usethesource.vallang.ISourceLocation; + +public class BreakTest { + + private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::ChangeScenarioTests"; + + public static void main(String[] args) throws IOException { + RascalShell.setupJavaProcessForREPL(); + + var term = RascalShell.connectToTerminal(); + var monitor = IRascalMonitor.buildConsoleMonitor(term); + var error = monitor instanceof PrintWriter ? (PrintWriter) monitor : new PrintWriter(System.err, false); + try { + for (int round = 0; round < 100; round++) { + if (runTest(monitor, error, "round-"+round)) { + break; + } + } + } finally { + error.close(); + } + } + + static boolean runTest(IRascalMonitor monitor, PrintWriter errorPrinter, String name) { + var output = new StringWriter(); + var failed = new AtomicBoolean(false); + try (var err = new PrintWriter(output, true); var out= new PrintWriter(output, false)) { + var projectRoot = RascalJUnitTestRunner.inferProjectRootFromClass(BreakTest.class); + var evaluator = ShellEvaluatorFactory.getDefaultEvaluatorForLocation(projectRoot, Reader.nullReader(), err, out, monitor, "$test-"+name+"$"); + evaluator.getConfiguration().setErrors(true); + // make sure we're writing to the outputs + evaluator.overwritePrintWriter(out, err); + + evaluator.job(name, 2, () -> { + evaluator.doNextImport(name, BREAKING_MODULE); + new TestEvaluator(evaluator, new ITestResultListener() { + + @Override + public void start(String context, int count) { + } + + @Override + public void report(boolean successful, String test, ISourceLocation loc, String message, + Throwable exception) { + if (!successful) { + failed.set(true); + err.println("!!!!! Failed test: " + test); + err.println("message: " + message); + } + } + + @Override + public void ignored(String test, ISourceLocation loc) { + } + + @Override + public void done() { + } + }).test(); + return null; + }); + } + + if (failed.get()) { + errorPrinter.println("❌❌❌ Test run failed: " + name); + errorPrinter.println("Job output:"); + errorPrinter.println(output.toString()); + return true; + } + return false; + } +} From e9bbe9d4f4f6aae16874899f2f1e3608c4d45daf Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Thu, 12 Mar 2026 12:35:00 +0100 Subject: [PATCH 02/19] Running tests in parallel --- .vscode/launch.json | 3 +- src/org/rascalmpl/compiler/BreakTest.java | 88 ++++++++++++------- .../lang/rascalcore/check/TestConfigs.rsc | 19 ++-- .../lang/rascalcore/check/TestShared.rsc | 8 ++ .../check/tests/BinaryDependencyTests.rsc | 5 +- .../check/tests/StaticTestingUtils.rsc | 8 +- 6 files changed, 83 insertions(+), 48 deletions(-) create mode 100644 src/org/rascalmpl/compiler/lang/rascalcore/check/TestShared.rsc diff --git a/.vscode/launch.json b/.vscode/launch.json index da94c795cbb..789e4e10b33 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,8 @@ "name": "BreakTest", "request": "launch", "mainClass": "org.rascalmpl.compiler.BreakTest", - "projectName": "rascal" + "projectName": "rascal", + "console": "integratedTerminal" }, { "type": "java", diff --git a/src/org/rascalmpl/compiler/BreakTest.java b/src/org/rascalmpl/compiler/BreakTest.java index 5a2ade2ac83..355fa61e615 100644 --- a/src/org/rascalmpl/compiler/BreakTest.java +++ b/src/org/rascalmpl/compiler/BreakTest.java @@ -5,31 +5,61 @@ import java.io.Reader; import java.io.StringWriter; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import org.rascalmpl.debug.IRascalMonitor; -import org.rascalmpl.interpreter.ITestResultListener; -import org.rascalmpl.interpreter.TestEvaluator; import org.rascalmpl.shell.RascalShell; import org.rascalmpl.shell.ShellEvaluatorFactory; import org.rascalmpl.test.infrastructure.RascalJUnitTestRunner; +import org.rascalmpl.uri.URIResolverRegistry; import io.usethesource.vallang.ISourceLocation; public class BreakTest { private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::ChangeScenarioTests"; + private static final String BREAKING_TEST = "fixedErrorsDisappear"; - public static void main(String[] args) throws IOException { + private static final int PARALLEL_RUNS = 8; // set to 1 to avoid any multi-threading interactions, but it might take 20 rounds or something + + public static void main(String[] args) throws IOException, InterruptedException { RascalShell.setupJavaProcessForREPL(); var term = RascalShell.connectToTerminal(); var monitor = IRascalMonitor.buildConsoleMonitor(term); var error = monitor instanceof PrintWriter ? (PrintWriter) monitor : new PrintWriter(System.err, false); try { - for (int round = 0; round < 100; round++) { - if (runTest(monitor, error, "round-"+round)) { - break; - } + AtomicBoolean failed = new AtomicBoolean(false); + AtomicInteger done = new AtomicInteger(0); + for (int t = 0; t < PARALLEL_RUNS; t++) { + var name = "T" + t; + var tr = new Thread(() -> { + try { + monitor.job(name, 100, (jobName, step) -> { + for (int round = 0; round < 100; round++) { + step.accept("round " + round, 1); + if (runTest(monitor, error, jobName +"-round-" + round)) { + failed.set(true); + System.exit(1); + break; + } + if (failed.get()) { + break; + + } + } + return null; + }); + } finally { + + done.incrementAndGet(); + } + }); + tr.start(); + } + while (done.get() < PARALLEL_RUNS && !failed.get()) { + Thread.sleep(100); } } finally { error.close(); @@ -46,34 +76,26 @@ static boolean runTest(IRascalMonitor monitor, PrintWriter errorPrinter, String // make sure we're writing to the outputs evaluator.overwritePrintWriter(out, err); - evaluator.job(name, 2, () -> { - evaluator.doNextImport(name, BREAKING_MODULE); - new TestEvaluator(evaluator, new ITestResultListener() { - - @Override - public void start(String context, int count) { - } - - @Override - public void report(boolean successful, String test, ISourceLocation loc, String message, - Throwable exception) { - if (!successful) { - failed.set(true); - err.println("!!!!! Failed test: " + test); - err.println("message: " + message); - } - } + evaluator.doImport(monitor, BREAKING_MODULE); - @Override - public void ignored(String test, ISourceLocation loc) { - } + try { + for (int i = 0; i < 10; i++) { + evaluator.call(BREAKING_TEST); + } + } catch (Throwable e ) { + failed.set(true); + err.println("❌ test fail "); + err.println(e); + } + // clean up memory + var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestShared"); + var testRoot = memoryModule.getFrameVariable("testRoot"); + try { + URIResolverRegistry.getInstance().remove((ISourceLocation)testRoot.getValue(), true); + } + catch (IOException e) { + } - @Override - public void done() { - } - }).test(); - return null; - }); } if (failed.get()) { diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc index e350c8902fc..1424d48e172 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc @@ -32,6 +32,7 @@ import lang::rascalcore::check::BasicRascalConfig; import lang::rascalcore::check::RascalConfig; import lang::rascalcore::check::ModuleLocations; +import lang::rascalcore::check::TestShared; // import lang::rascalcore::CompilerPathConfig; // Duplicate in lang::rascalcore::compile::util::Names, factor out @@ -74,10 +75,10 @@ public PathConfig getDefaultTestingPathConfig() { npc += 1; snpc = ""; return pathConfig( - srcs = [ |memory:///test-modules/|, |std:///| ], - bin = |memory:///test-modules/rascal-tests-bin-|, - generatedSources = |memory:///test-modules/generated-test-sources-|, - generatedResources = |memory:///test-modules/generated-test-resources-|, + srcs = [ testModulesRoot, |std:///| ], + bin = testModulesRoot + "rascal-tests-bin-", + generatedSources = testModulesRoot + "generated-test-sources-", + generatedResources = testModulesRoot + "generated-test-resources-", libs = [ ] ); } @@ -92,10 +93,10 @@ public PathConfig getReleasedStandardLibraryTestingPathConfig() { npc += 1; snpc = ""; return pathConfig( - srcs = [ |memory:///test-modules/| ], - bin = |memory:///test-modules/rascal-tests-bin-|, - generatedSources = |memory:///test-modules/generated-test-sources-|, - generatedResources = |memory:///test-modules/generated-test-resources-|, + srcs = [ testModulesRoot ], + bin = testModulesRoot + "rascal-tests-bin-", + generatedSources = testModulesRoot + "generated-test-sources-", + generatedResources = testModulesRoot + "generated-test-resources-", libs = [ |lib://rascal| ] ); } @@ -108,7 +109,7 @@ public PathConfig getRascalProjectTestingPathConfig() { snpc = ""; return pathConfig( srcs = [|project://rascal/src/org/rascalmpl/library|], - bin = |memory:///test-modules/rascal-lib-bin-|, + bin = testModulesRoot + "rascal-lib-bin-", libs = [] ); } diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestShared.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestShared.rsc new file mode 100644 index 00000000000..b03c9628e45 --- /dev/null +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestShared.rsc @@ -0,0 +1,8 @@ +module lang::rascalcore::check::TestShared + +import util::UUID; + + +public loc testRoot = uuid()[scheme="memory"]; +public loc testModulesRoot = testRoot + "test-modules"; + diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc index 5bf5c69c708..e4c47d642f6 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc @@ -46,6 +46,7 @@ import String; import lang::rascalcore::check::ModuleLocations; import util::FileSystem; import util::SemVer; +import lang::rascalcore::check::TestShared; // ---- Utilities for test setup ---------------------------------------------- @@ -57,10 +58,10 @@ data PathConfig(loc generatedResources=|unknown:///|, loc generatedSources=|unkn data Project = project(str name, map[str moduleName, str moduleText] modules, PathConfig pcfg); -void clearMemory() { remove(|memory:///|, recursive = true); } +void clearMemory() { remove(testRoot, recursive = true); } loc projectDir(str pname) - = |memory:///|; + = testRoot + pname; loc src(str pname) = projectDir(pname) + "src/"; diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc index b64028e1f68..5d2c2eeca53 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc @@ -44,6 +44,8 @@ import util::Reflective; import ParseTree; import lang::rascalcore::check::RascalConfig; +import lang::rascalcore::check::TestShared; + import lang::rascalcore::check::Checker; import lang::rascal::\syntax::Rascal; @@ -81,14 +83,14 @@ loc composeModule(str stmts){ } void clearMemory() { - remove(|memory:///test-modules/| recursive = true); + remove(testModulesRoot, recursive = true); } str cleanName(str name) = name[0] == "\\" ? name[1..] : name; loc writeModule(str moduleText){ = extractModuleNameAndBody(moduleText); - mloc = |memory:///test-modules/.rsc|; + mloc = testModulesRoot + ".rsc"; writeFile(mloc, moduleText); return mloc; } @@ -99,7 +101,7 @@ list[loc] writeModules(str modules...) void removeModule(str mname){ pcfg = getDefaultTestingPathConfig(); name = cleanName(mname); - remove(|memory:///test-modules/.rsc|); + remove(testModulesRoot + ".rsc"); remove(pcfg.generatedResources + ".tpl"); } From fbf1836659aee71e83adda27db0a32717858c81b Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Thu, 12 Mar 2026 17:11:57 +0100 Subject: [PATCH 03/19] Making the error case faster to reproduce and reduced the error size --- src/org/rascalmpl/compiler/BreakTest.java | 80 ++++++++++--------- .../check/tests/ChangeScenarioTests.rsc | 23 ++++++ 2 files changed, 67 insertions(+), 36 deletions(-) diff --git a/src/org/rascalmpl/compiler/BreakTest.java b/src/org/rascalmpl/compiler/BreakTest.java index 355fa61e615..3610fb61b02 100644 --- a/src/org/rascalmpl/compiler/BreakTest.java +++ b/src/org/rascalmpl/compiler/BreakTest.java @@ -4,6 +4,8 @@ import java.io.PrintWriter; import java.io.Reader; import java.io.StringWriter; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -19,9 +21,10 @@ public class BreakTest { private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::ChangeScenarioTests"; - private static final String BREAKING_TEST = "fixedErrorsDisappear"; + private static final String BREAKING_TEST = "fixedErrorsDisappear2"; private static final int PARALLEL_RUNS = 8; // set to 1 to avoid any multi-threading interactions, but it might take 20 rounds or something + private static final int TRIES = 1000 / PARALLEL_RUNS; public static void main(String[] args) throws IOException, InterruptedException { RascalShell.setupJavaProcessForREPL(); @@ -33,26 +36,19 @@ public static void main(String[] args) throws IOException, InterruptedException AtomicBoolean failed = new AtomicBoolean(false); AtomicInteger done = new AtomicInteger(0); for (int t = 0; t < PARALLEL_RUNS; t++) { - var name = "T" + t; + var name = "Thread " + (t + 1); var tr = new Thread(() -> { try { - monitor.job(name, 100, (jobName, step) -> { - for (int round = 0; round < 100; round++) { - step.accept("round " + round, 1); - if (runTest(monitor, error, jobName +"-round-" + round)) { - failed.set(true); - System.exit(1); - break; - } - if (failed.get()) { - break; - - } - } - return null; - }); + if (crashTest(monitor, error, name, failed)) { + failed.set(true); + System.err.println("We got a failure, exiting now!"); + Thread.sleep(1000); + System.exit(1); + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); } finally { - done.incrementAndGet(); } }); @@ -66,9 +62,9 @@ public static void main(String[] args) throws IOException, InterruptedException } } - static boolean runTest(IRascalMonitor monitor, PrintWriter errorPrinter, String name) { + static boolean crashTest(IRascalMonitor monitor, PrintWriter errorPrinter, String name, AtomicBoolean failed) { var output = new StringWriter(); - var failed = new AtomicBoolean(false); + var iFailed = new AtomicBoolean(false); try (var err = new PrintWriter(output, true); var out= new PrintWriter(output, false)) { var projectRoot = RascalJUnitTestRunner.inferProjectRootFromClass(BreakTest.class); var evaluator = ShellEvaluatorFactory.getDefaultEvaluatorForLocation(projectRoot, Reader.nullReader(), err, out, monitor, "$test-"+name+"$"); @@ -77,28 +73,40 @@ static boolean runTest(IRascalMonitor monitor, PrintWriter errorPrinter, String evaluator.overwritePrintWriter(out, err); evaluator.doImport(monitor, BREAKING_MODULE); - try { - for (int i = 0; i < 10; i++) { - evaluator.call(BREAKING_TEST); + monitor.job(name, TRIES, (jobname, step) -> { + try { + for (int i = 0; i < TRIES; i++) { + if (failed.get()) { + return false; + } + monitor.jobStep(jobname, "Running: try " + (i + 1)); + evaluator.call(BREAKING_TEST); + } + + } catch (Throwable e ) { + failed.set(true); + iFailed.set(true); + err.println("❌ test fail "); + err.println(e); + } + return null; + }); + } finally { + // clean up memory + var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestShared"); + var testRoot = memoryModule.getFrameVariable("testRoot"); + try { + URIResolverRegistry.getInstance().remove((ISourceLocation)testRoot.getValue(), true); + } + catch (Throwable e) { + err.println("Failure to cleanup the cache"); } - } catch (Throwable e ) { - failed.set(true); - err.println("❌ test fail "); - err.println(e); - } - // clean up memory - var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestShared"); - var testRoot = memoryModule.getFrameVariable("testRoot"); - try { - URIResolverRegistry.getInstance().remove((ISourceLocation)testRoot.getValue(), true); - } - catch (IOException e) { } } - if (failed.get()) { + if (iFailed.get()) { errorPrinter.println("❌❌❌ Test run failed: " + name); errorPrinter.println("Job output:"); errorPrinter.println(output.toString()); diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc index 00c5b855b5b..afb277fa129 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc @@ -469,6 +469,29 @@ test bool breakingChange1(){ return expectReChecks(D, ["C", "D"]); } +test bool fixedErrorsDisappear2() { // ht @toinehartman + clearMemory(); + pcfg = getDefaultTestingPathConfig(); + + mlocs = writeModules(" + module ParserBase + "); + + assert checkModulesOK(mlocs, pathConfig=pcfg) : "Precondition failed: no errors expected!"; + + // Introduce a type error (import of module that does not exist) + l = writeModule(" + module ParserBase + + import vis::ParseTree; // module does not exist -\> error + + "); + + assert missingModuleInModule(l, pathConfig=pcfg) : "Precondition failed: expected at least one error, but got none!"; + + return true; +} + test bool fixedErrorsDisappear() { // ht @toinehartman clearMemory(); pcfg = getReleasedStandardLibraryTestingPathConfig(); From b21263060b10cc93ed92d88fdb2772b29e72a929 Mon Sep 17 00:00:00 2001 From: paulklint Date: Fri, 13 Mar 2026 21:30:38 +0100 Subject: [PATCH 04/19] Added print utility --- .../rascalcore/check/tests/StaticTestingUtils.rsc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc index 5d2c2eeca53..cc6f520830e 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc @@ -42,6 +42,7 @@ import Relation; import Set; import util::Reflective; import ParseTree; +import util::FileSystem; import lang::rascalcore::check::RascalConfig; import lang::rascalcore::check::TestShared; @@ -105,6 +106,16 @@ void removeModule(str mname){ remove(pcfg.generatedResources + ".tpl"); } +void printModules(){ + for(f <- find(testModulesRoot, "rsc")){ + println(" : + '"); + } + for(f <- find(testModulesRoot, "tpl")){ + println(": "); + } +} + set[Message] getErrorMessages(ModuleStatus r) = { m | m <- getAllMessages(r), m is error }; From 2c8032f4a2aa3590d2cd904162391dde9d4b2b83 Mon Sep 17 00:00:00 2001 From: paulklint Date: Fri, 13 Mar 2026 21:31:08 +0100 Subject: [PATCH 05/19] Root cause of this issues lmMloc > lmTpl should be lmMloc >= lmTpl; --- .../compiler/lang/rascalcore/check/CheckerCommon.rsc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/CheckerCommon.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/CheckerCommon.rsc index 592e642e3ed..b9df1cb0fe3 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/CheckerCommon.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/CheckerCommon.rsc @@ -224,8 +224,8 @@ bool tplOutdated(MODID moduleId, PathConfig pcfg){ = getTPLReadLoc(qualifiedModuleName, pcfg); lmMloc = lastModified(mloc); lmTpl = lastModified(tpl); - res = !found || lmMloc > lmTpl; - //println("tplOutdated : ; mloc: \> tpl: : lmTpl>, (, )"); + res = !found || lmMloc >= lmTpl; + // println("tplOutdated : ; mloc: \> tpl: : lmTpl>, (, )"); return res; } catch _: { return false; From 131bf50a76de781e61d027a5cb1c1cd8b13610de Mon Sep 17 00:00:00 2001 From: paulklint Date: Fri, 13 Mar 2026 21:32:08 +0100 Subject: [PATCH 06/19] Auxiliary safety measure. --- src/org/rascalmpl/compiler/lang/rascalcore/check/Checker.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/Checker.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/Checker.rsc index c3ade402c51..37a3e4e7c56 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/Checker.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/Checker.rsc @@ -553,7 +553,7 @@ bool uptodateTPls(list[loc] candidates, list[str] mnames, PathConfig pcfg){ for(int i <- index(candidates)){ mloc = candidates[i]; = getTPLReadLoc(mnames[i], pcfg); - if(!found || lastModified(mloc) > lastModified(tpl)){ + if(!found || lastModified(mloc) >= lastModified(tpl)){ return false; } } From f7d96a9071b3ddb50321f8058998fd023affba12 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 17 Mar 2026 17:41:42 +0100 Subject: [PATCH 07/19] Working on breaking more tests --- src/org/rascalmpl/compiler/BreakTest.java | 114 +++++++++++++++++- .../check/tests/ChangeScenarioTests.rsc | 14 +++ .../rascalcore/check/tests/PackagerTests.rsc | 2 +- .../lang/rascalcore/check/tests/RunTests.java | 6 +- 4 files changed, 129 insertions(+), 7 deletions(-) diff --git a/src/org/rascalmpl/compiler/BreakTest.java b/src/org/rascalmpl/compiler/BreakTest.java index 3610fb61b02..ab0946340ef 100644 --- a/src/org/rascalmpl/compiler/BreakTest.java +++ b/src/org/rascalmpl/compiler/BreakTest.java @@ -4,24 +4,35 @@ import java.io.PrintWriter; import java.io.Reader; import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.rascalmpl.debug.IRascalMonitor; +import org.rascalmpl.interpreter.Evaluator; +import org.rascalmpl.interpreter.result.AbstractFunction; import org.rascalmpl.shell.RascalShell; import org.rascalmpl.shell.ShellEvaluatorFactory; import org.rascalmpl.test.infrastructure.RascalJUnitTestRunner; import org.rascalmpl.uri.URIResolverRegistry; +import io.usethesource.vallang.IBool; import io.usethesource.vallang.ISourceLocation; public class BreakTest { - private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::ChangeScenarioTests"; - private static final String BREAKING_TEST = "fixedErrorsDisappear2"; + // private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::ChangeScenarioTests"; + // private static final String BREAKING_TEST = "fixedErrorsDisappear2"; + private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::BinaryDependencyTests"; + private static final String BREAKING_TEST = "notCompatibleAfterRemovingConstructor"; + + // notCompatibleAfterChangingFunctionArgument + // notCompatibleAfterChangingFunctionArgument private static final int PARALLEL_RUNS = 8; // set to 1 to avoid any multi-threading interactions, but it might take 20 rounds or something private static final int TRIES = 1000 / PARALLEL_RUNS; @@ -75,19 +86,38 @@ static boolean crashTest(IRascalMonitor monitor, PrintWriter errorPrinter, Strin evaluator.doImport(monitor, BREAKING_MODULE); try { monitor.job(name, TRIES, (jobname, step) -> { + String currentTest = ""; + var tests = evaluator.getHeap().getModule(BREAKING_MODULE).getTests(); try { + tests = tests.stream() + .filter(t -> !t.hasTag("ignore")) + .collect(Collectors.toList()); + + monitor.jobTodo(jobname, tests.size() * TRIES); + for (int i = 0; i < TRIES; i++) { if (failed.get()) { return false; } + + Collections.shuffle(tests); + monitor.jobStep(jobname, "Running: try " + (i + 1)); - evaluator.call(BREAKING_TEST); + for (var t: tests) { + currentTest = t.getName(); + monitor.jobStep(jobname, "Running: try " + (i + 1) + " => " + currentTest); + var result = t.call(); + if (!((IBool)result).getValue()) { + throw new RuntimeException("Test " + currentTest +" returned false"); + } + } } } catch (Throwable e ) { failed.set(true); iFailed.set(true); - err.println("❌ test fail "); + err.println("tests" + tests.stream().map(AbstractFunction::getName).toArray()); + err.println("❌ test fail :" + currentTest); err.println(e); } return null; @@ -114,4 +144,80 @@ static boolean crashTest(IRascalMonitor monitor, PrintWriter errorPrinter, Strin } return false; } + + static boolean crashTestFreshEval(IRascalMonitor monitor, PrintWriter errorPrinter, String name, AtomicBoolean failed) { + var output = new StringWriter(); + var iFailed = new AtomicBoolean(false); + try (var err = new PrintWriter(output, true); var out= new PrintWriter(output, false)) { + var projectRoot = RascalJUnitTestRunner.inferProjectRootFromClass(BreakTest.class); + monitor.job(name, TRIES * 2, (jobname, step) -> { + String currentTest = ""; + var tests = Collections.emptyList(); + try { + for (int i = 0; i < TRIES; i++) { + if (failed.get()) { + return false; + } + monitor.jobStep(jobname, "Try " + (i + 1)); + var evaluator = ShellEvaluatorFactory.getDefaultEvaluatorForLocation(projectRoot, Reader.nullReader(), err, out, monitor, "$test-"+name+"$"); + try { + + evaluator.getConfiguration().setErrors(true); + // make sure we're writing to the outputs + evaluator.overwritePrintWriter(out, err); + + evaluator.doImport(monitor, BREAKING_MODULE); + + tests = evaluator.getHeap().getModule(BREAKING_MODULE).getTests() + .stream() + .filter(t -> !t.hasTag("ignore")) + .collect(Collectors.toList()); + + if (i == 0) { + monitor.jobTodo(jobname, tests.size() * TRIES); + } + + Collections.shuffle(tests); + + monitor.jobStep(jobname, "Running: try " + (i + 1)); + for (var t: tests) { + currentTest = t.getName(); + monitor.jobStep(jobname, "Running: try " + (i + 1) + " => " + currentTest); + var result = t.call(); + if (!((IBool)result).getValue()) { + throw new RuntimeException("Test " + currentTest +" returned false"); + } + } + } finally { + // clean up memory + var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestShared"); + var testRoot = memoryModule.getFrameVariable("testRoot"); + try { + URIResolverRegistry.getInstance().remove((ISourceLocation)testRoot.getValue(), true); + } + catch (Throwable e) { + err.println("Failure to cleanup the cache"); + } + } + } + + } catch (Throwable e ) { + failed.set(true); + iFailed.set(true); + err.println("tests: " + Arrays.deepToString(tests.stream().map(AbstractFunction::getName).toArray())); + err.println("❌ test fail :" + currentTest); + err.println(e); + } + return null; + }); + } + + if (iFailed.get()) { + errorPrinter.println("❌❌❌ Test run failed: " + name); + errorPrinter.println("Job output:"); + errorPrinter.println(output.toString()); + return true; + } + return false; + } } diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc index afb277fa129..7b6c16459f6 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc @@ -469,6 +469,20 @@ test bool breakingChange1(){ return expectReChecks(D, ["C", "D"]); } +test bool timestampsIncrease() { + datetime lastTime = $2020-01-01T00:00:00.000+00:00$; + loc testLoc = |memory:///test-random-write-file|; + for (i <- [0..10000]) { + writeFile(testLoc, "Hoi"); + datetime newTime = lastModified(testLoc); + if (lastTime == newTime) { + println("Time did not change: == "); + return false; + } + } + return true; +} + test bool fixedErrorsDisappear2() { // ht @toinehartman clearMemory(); pcfg = getDefaultTestingPathConfig(); diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/PackagerTests.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/PackagerTests.rsc index 62902a84590..ee153433ed7 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/PackagerTests.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/PackagerTests.rsc @@ -48,7 +48,7 @@ private void runChecker(PathConfig pcfg, list[str] mnames) { test bool packagerRewritesTModelsCorrectly () { println("**** Checking List"); - tplRoot = |memory:///|; + tplRoot = |memory://packager-test/|; originalBin = tplRoot + "rascal-lib"; rewrittenBin = tplRoot + "rascal-lib-rewritten"; rascalLibPcfg = diff --git a/test/org/rascalmpl/compiler/lang/rascalcore/check/tests/RunTests.java b/test/org/rascalmpl/compiler/lang/rascalcore/check/tests/RunTests.java index 05cf4b7b415..26e6a6591bf 100644 --- a/test/org/rascalmpl/compiler/lang/rascalcore/check/tests/RunTests.java +++ b/test/org/rascalmpl/compiler/lang/rascalcore/check/tests/RunTests.java @@ -27,11 +27,13 @@ package org.rascalmpl.compiler.lang.rascalcore.check.tests; import org.junit.runner.RunWith; +import org.rascalmpl.test.infrastructure.RascalJUnitParallelRecursiveTestRunner; import org.rascalmpl.test.infrastructure.RascalJUnitTestPrefix; import org.rascalmpl.test.infrastructure.RascalJUnitTestRunner; +import org.rascalmpl.test.infrastructure.RecursiveRascalParallelTest; -@RunWith(RascalJUnitTestRunner.class) -@RascalJUnitTestPrefix("lang::rascalcore::check::tests") +@RunWith(RascalJUnitParallelRecursiveTestRunner.class) +@RecursiveRascalParallelTest("lang::rascalcore::check::tests") public class RunTests { } From dd75241f18bb7aeca237ae1bde89cedb98112192 Mon Sep 17 00:00:00 2001 From: paulklint Date: Thu, 19 Mar 2026 14:41:14 +0100 Subject: [PATCH 08/19] Commented out println --- .../rascalmpl/compiler/lang/rascalcore/check/ADTandGrammar.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/ADTandGrammar.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/ADTandGrammar.rsc index e60eb823db1..1f2f3564282 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/ADTandGrammar.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/ADTandGrammar.rsc @@ -178,7 +178,7 @@ tuple[bool, TModel, ModuleStatus] addGrammar(MODID moduleId, set[MODID] imports, = getTModelForModule(m, ms); if(!found) { msg = error("Cannot add grammar or tmodel since `` is not found", ms.moduleLocs[moduleId] ? |unknown:///|); - println(msg); // TODO: Just to record this event; this should probably go to a log file + //println(msg); // TODO: Just to record this event; this should probably go to a log file ms.messages[moduleId] ? {} += { msg }; tm1 = tmodel(modelName=qualifiedModuleName, messages=[msg]); return ; From 5e05727b33c147229b3a087a21afc6a4ed9c7704 Mon Sep 17 00:00:00 2001 From: paulklint Date: Thu, 19 Mar 2026 14:42:18 +0100 Subject: [PATCH 09/19] Undo previous > to >= change --- .../rascalmpl/compiler/lang/rascalcore/check/CheckerCommon.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/CheckerCommon.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/CheckerCommon.rsc index b9df1cb0fe3..b8f89b62721 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/CheckerCommon.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/CheckerCommon.rsc @@ -224,7 +224,7 @@ bool tplOutdated(MODID moduleId, PathConfig pcfg){ = getTPLReadLoc(qualifiedModuleName, pcfg); lmMloc = lastModified(mloc); lmTpl = lastModified(tpl); - res = !found || lmMloc >= lmTpl; + res = !found || lmMloc > lmTpl; // println("tplOutdated : ; mloc: \> tpl: : lmTpl>, (, )"); return res; } catch _: { From ef2b8625b5bd00b3fa54197dbe299820061f8412 Mon Sep 17 00:00:00 2001 From: paulklint Date: Thu, 19 Mar 2026 14:43:25 +0100 Subject: [PATCH 10/19] Improved computation of require definitions --- .../compiler/lang/rascalcore/check/Import.rsc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/Import.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/Import.rsc index 220ba9e2d21..103f5c28c85 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/Import.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/Import.rsc @@ -328,10 +328,11 @@ str getModuleNameFromAnyLogical(loc l){ tuple[bool, ModuleStatus] importsAndExtendsAreBinaryCompatible(TModel tm, set[MODID] importsAndExtends, ModuleStatus ms){ moduleName = tm.modelName; physical2logical = invertUnique(tm.logical2physical); - - modRequires = { lg | l <- range(tm.useDef), - physical2logical[l]?, lg := physical2logical[l], - moduleName !:= getModuleNameFromAnyLogical(lg) }; + modRequires = {lg | l <- range(tm.useDef), + l in physical2logical, + lg := physical2logical[l], + moduleName != getModuleNameFromAnyLogical(lg) + }; provided = {}; if(!isEmpty(modRequires)){ for(m <- importsAndExtends){ @@ -342,8 +343,6 @@ tuple[bool, ModuleStatus] importsAndExtendsAreBinaryCompatible(TModel tm, set[MO } } - //println(" requires "); - if(isEmpty(modRequires - provided)){ //println("importsAndExtendsAreBinaryCompatible : satisfied"); return ; From 1453d6c1c101e33f9f32255d21789bb8948f588f Mon Sep 17 00:00:00 2001 From: paulklint Date: Thu, 19 Mar 2026 14:44:52 +0100 Subject: [PATCH 11/19] Small refactoring and added asserts --- .../check/tests/BinaryDependencyTests.rsc | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc index e4c47d642f6..11ba7cf8f51 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc @@ -47,6 +47,7 @@ import lang::rascalcore::check::ModuleLocations; import util::FileSystem; import util::SemVer; import lang::rascalcore::check::TestShared; +import lang::rascalcore::check::tests::StaticTestingUtils; // ---- Utilities for test setup ---------------------------------------------- @@ -100,6 +101,9 @@ str writeModule(str mname, str mtext){ throw "Parse error in "; } +loc getModuleLoc(str mname, Project pd) + = src(pd.name) + ".rsc"; + PathConfig createPathConfig(str pname){ return pathConfig( srcs=[src(pname)], @@ -112,7 +116,9 @@ PathConfig createPathConfig(str pname){ Project addModule(str mname, str mtext, Project pd){ pd.modules[mname] = writeModule(mname, mtext); - writeFile(src(pd.name) + ".rsc", pd.modules[mname]); + mloc = getModuleLoc(mname, pd); + writeFile(mloc, pd.modules[mname]); + assert exists(mloc) : " does not exist after write"; return pd; } @@ -120,14 +126,18 @@ Project changeModule(str mname, str mtext, Project pd){ if(!pd.modules[mname]?) throw "Module does not exist in "; pd.modules[mname] = writeModule(mname, mtext); - writeFile(src(pd.name) + ".rsc", pd.modules[mname]); + mloc = getModuleLoc(mname, pd); + writeFile(mloc, pd.modules[mname]); + assert exists(mloc) : " does not exist after write"; return pd; } Project removeSourceOfModule(str mname, Project pd){ if(!pd.modules[mname]?) throw "Cannot remove non-existing module "; pd.modules = delete(pd.modules, mname); - remove(src(pd.name) + ".rsc", recursive=true); + mloc = getModuleLoc(mname, pd); + remove(mloc, recursive=true); + assert !exists(mloc): " not removed"; return pd; } @@ -534,7 +544,7 @@ test bool incompatibleVersionsOfBinaryLibrary(){ // Important: we do not recompile TP (and thus it will contain the outdated version of IO) - // Update Checks' modification time to make sure it will rechecked + // Update Checks' modification time to make sure it will be rechecked touch(getRascalModuleLocation("Check", core.pcfg)); // Recompile Check and discover the error return checkExpectErrors("Check", ["Review of dependencies, reconfiguration or recompilation needed: binary module `TP` depends (indirectly) on incompatible module(s)"], core.pcfg, remove = [rascal, typepal, core]); From b28fd56ab96b6ef17d7803a1a07cf92f05e34154 Mon Sep 17 00:00:00 2001 From: paulklint Date: Thu, 19 Mar 2026 14:46:01 +0100 Subject: [PATCH 12/19] Resolved testRoot/testModulesRoot confusion --- .../compiler/lang/rascalcore/check/TestConfigs.rsc | 12 ++++++------ .../compiler/lang/rascalcore/check/TestShared.rsc | 2 +- .../rascalcore/check/tests/StaticTestingUtils.rsc | 8 +++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc index 1424d48e172..a4bc0080a82 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc @@ -76,9 +76,9 @@ public PathConfig getDefaultTestingPathConfig() { snpc = ""; return pathConfig( srcs = [ testModulesRoot, |std:///| ], - bin = testModulesRoot + "rascal-tests-bin-", - generatedSources = testModulesRoot + "generated-test-sources-", - generatedResources = testModulesRoot + "generated-test-resources-", + bin = testRoot + "rascal-tests-bin-", + generatedSources = testRoot + "generated-test-sources-", + generatedResources = testRoot + "generated-test-resources-", libs = [ ] ); } @@ -94,9 +94,9 @@ public PathConfig getReleasedStandardLibraryTestingPathConfig() { snpc = ""; return pathConfig( srcs = [ testModulesRoot ], - bin = testModulesRoot + "rascal-tests-bin-", - generatedSources = testModulesRoot + "generated-test-sources-", - generatedResources = testModulesRoot + "generated-test-resources-", + bin = testRoot + "rascal-tests-bin-", + generatedSources = testRoot + "generated-test-sources-", + generatedResources = testRoot + "generated-test-resources-", libs = [ |lib://rascal| ] ); } diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestShared.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestShared.rsc index b03c9628e45..ac347670a50 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestShared.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestShared.rsc @@ -4,5 +4,5 @@ import util::UUID; public loc testRoot = uuid()[scheme="memory"]; -public loc testModulesRoot = testRoot + "test-modules"; +public loc testModulesRoot = testRoot + "src"; diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc index cc6f520830e..1d824d3ee83 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc @@ -84,7 +84,7 @@ loc composeModule(str stmts){ } void clearMemory() { - remove(testModulesRoot, recursive = true); + remove(testRoot, recursive = true); } str cleanName(str name) = name[0] == "\\" ? name[1..] : name; @@ -107,13 +107,15 @@ void removeModule(str mname){ } void printModules(){ - for(f <- find(testModulesRoot, "rsc")){ + println("\<\<\<\<"); + for(f <- find(testRoot, "rsc")){ println(" : '"); } - for(f <- find(testModulesRoot, "tpl")){ + for(f <- find(testRoot, "tpl")){ println(": "); } + println("\>\>\>\>"); } set[Message] getErrorMessages(ModuleStatus r) From e133fde8fbb46fb2dfd70f93603ccc95d6363646 Mon Sep 17 00:00:00 2001 From: paulklint Date: Fri, 20 Mar 2026 15:11:59 +0100 Subject: [PATCH 13/19] Increased decoupling between individual tests --- .../compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc index 1d824d3ee83..9430c36cb7d 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc @@ -76,7 +76,7 @@ tuple[str,str] extractModuleNameAndBody(str moduleText){ loc composeModule(str stmts){ return writeModule( - "module TestModule + "module TestModule 'value main(){ ' \n ' return true; From 23119a7b53702ce7f795c4184a3bf4b2cad327c5 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 23 Mar 2026 12:28:24 +0100 Subject: [PATCH 14/19] Make sure files cannot race on their timestamp --- .../uri/libraries/MemoryResolver.java | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/org/rascalmpl/uri/libraries/MemoryResolver.java b/src/org/rascalmpl/uri/libraries/MemoryResolver.java index c174c9ae370..3f14739f3e5 100644 --- a/src/org/rascalmpl/uri/libraries/MemoryResolver.java +++ b/src/org/rascalmpl/uri/libraries/MemoryResolver.java @@ -25,6 +25,8 @@ import java.nio.file.NoSuchFileException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; + import org.checkerframework.checker.nullness.qual.Nullable; import org.rascalmpl.uri.FileAttributes; import org.rascalmpl.uri.ISourceLocationInputOutput; @@ -62,8 +64,29 @@ */ public class MemoryResolver implements ISourceLocationInputOutput { + private static class MemoryEntry extends FSEntry { + /** keep track of the timestamp across entries */ + private static final AtomicLong now = new AtomicLong(System.currentTimeMillis()); + + /** + * Every time we're called, we get a new timestamp, to make sure file written at roughly the same + * time don't get the same modification timestamp + * @return + */ + private static long timestamp() { + return now.accumulateAndGet(System.currentTimeMillis(), (now, prev) -> { + if (now > prev) { + return now; + } + if (now == prev) { + return now + 1; + } + // else : prev > now + return prev + 1; + }); + } private final @Nullable byte[] contents; public MemoryEntry() { @@ -71,7 +94,11 @@ public MemoryEntry() { } public MemoryEntry(@Nullable byte[] contents) { - this(System.currentTimeMillis(), System.currentTimeMillis(), contents); + this(timestamp(), contents); + } + + private MemoryEntry(long created, @Nullable byte[] contents) { + this(created, created, contents); } public MemoryEntry(long created, long lastModified) { @@ -83,19 +110,11 @@ public MemoryEntry(long created, long lastModified, @Nullable byte[] contents) { } public MemoryEntry newContents(byte[] newContents) { - long newTimestamp = System.currentTimeMillis(); - if (newTimestamp <= getLastModified()) { - newTimestamp = getLastModified() + 1; - } - return new MemoryEntry(getCreated(), newTimestamp, newContents); + return new MemoryEntry(getCreated(), timestamp(), newContents); } public MemoryEntry copy() { - long newTimestamp = System.currentTimeMillis(); - if (newTimestamp <= getLastModified()) { - newTimestamp = getLastModified() + 1; - } - return new MemoryEntry(newTimestamp, newTimestamp, contents); + return new MemoryEntry(timestamp(), contents); } } From 511bb7b886243c14472d100238d2aaf5345cd635 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 23 Mar 2026 12:28:56 +0100 Subject: [PATCH 15/19] Trying to get errors to print --- src/org/rascalmpl/compiler/BreakTest.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/org/rascalmpl/compiler/BreakTest.java b/src/org/rascalmpl/compiler/BreakTest.java index ab0946340ef..0114273366e 100644 --- a/src/org/rascalmpl/compiler/BreakTest.java +++ b/src/org/rascalmpl/compiler/BreakTest.java @@ -28,8 +28,8 @@ public class BreakTest { // private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::ChangeScenarioTests"; // private static final String BREAKING_TEST = "fixedErrorsDisappear2"; - private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::BinaryDependencyTests"; - private static final String BREAKING_TEST = "notCompatibleAfterRemovingConstructor"; + private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::FunctionTCTests"; + private static final String BREAKING_TEST = "BoundOK1"; // notCompatibleAfterChangingFunctionArgument // notCompatibleAfterChangingFunctionArgument @@ -43,8 +43,8 @@ public static void main(String[] args) throws IOException, InterruptedException var term = RascalShell.connectToTerminal(); var monitor = IRascalMonitor.buildConsoleMonitor(term); var error = monitor instanceof PrintWriter ? (PrintWriter) monitor : new PrintWriter(System.err, false); + AtomicBoolean failed = new AtomicBoolean(false); try { - AtomicBoolean failed = new AtomicBoolean(false); AtomicInteger done = new AtomicInteger(0); for (int t = 0; t < PARALLEL_RUNS; t++) { var name = "Thread " + (t + 1); @@ -52,9 +52,9 @@ public static void main(String[] args) throws IOException, InterruptedException try { if (crashTest(monitor, error, name, failed)) { failed.set(true); + error.flush(); System.err.println("We got a failure, exiting now!"); Thread.sleep(1000); - System.exit(1); } } catch (InterruptedException e) { @@ -69,8 +69,14 @@ public static void main(String[] args) throws IOException, InterruptedException Thread.sleep(100); } } finally { + error.flush(); error.close(); } + if (failed.get()) { + System.out.flush(); + System.err.flush(); + System.exit(1); + } } static boolean crashTest(IRascalMonitor monitor, PrintWriter errorPrinter, String name, AtomicBoolean failed) { @@ -116,7 +122,6 @@ static boolean crashTest(IRascalMonitor monitor, PrintWriter errorPrinter, Strin } catch (Throwable e ) { failed.set(true); iFailed.set(true); - err.println("tests" + tests.stream().map(AbstractFunction::getName).toArray()); err.println("❌ test fail :" + currentTest); err.println(e); } From 083292b5adb49b8417c4f84ceee018436571b695 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 23 Mar 2026 13:50:39 +0100 Subject: [PATCH 16/19] Using fresh eval every time --- src/org/rascalmpl/compiler/BreakTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/compiler/BreakTest.java b/src/org/rascalmpl/compiler/BreakTest.java index 0114273366e..0d0cd12fe31 100644 --- a/src/org/rascalmpl/compiler/BreakTest.java +++ b/src/org/rascalmpl/compiler/BreakTest.java @@ -50,7 +50,7 @@ public static void main(String[] args) throws IOException, InterruptedException var name = "Thread " + (t + 1); var tr = new Thread(() -> { try { - if (crashTest(monitor, error, name, failed)) { + if (crashTestFreshEval(monitor, error, name, failed)) { failed.set(true); error.flush(); System.err.println("We got a failure, exiting now!"); From 54fe604e7f8398039badd4ba4f112ba7847e7de2 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 23 Mar 2026 15:22:32 +0100 Subject: [PATCH 17/19] Fixed a path config bin dir --- .../rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc index 594ac046c52..6e5c907c847 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc @@ -109,7 +109,7 @@ public PathConfig getRascalProjectTestingPathConfig() { snpc = ""; return pathConfig( srcs = [|project://rascal/src/org/rascalmpl/library|], - bin = testModulesRoot + "rascal-lib-bin-", + bin = testRoot + "rascal-lib-bin-", libs = [] ); } From 4d3e53dfdfb80ced2d6339212a0df05a03a2dce4 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 23 Mar 2026 15:43:03 +0100 Subject: [PATCH 18/19] Make sure error messages are not lost --- src/org/rascalmpl/compiler/BreakTest.java | 27 ++++++++++++++--------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/org/rascalmpl/compiler/BreakTest.java b/src/org/rascalmpl/compiler/BreakTest.java index 0d0cd12fe31..cdbcd34394d 100644 --- a/src/org/rascalmpl/compiler/BreakTest.java +++ b/src/org/rascalmpl/compiler/BreakTest.java @@ -34,7 +34,7 @@ public class BreakTest { // notCompatibleAfterChangingFunctionArgument // notCompatibleAfterChangingFunctionArgument - private static final int PARALLEL_RUNS = 8; // set to 1 to avoid any multi-threading interactions, but it might take 20 rounds or something + private static final int PARALLEL_RUNS = 1; // set to 1 to avoid any multi-threading interactions, but it might take 20 rounds or something private static final int TRIES = 1000 / PARALLEL_RUNS; public static void main(String[] args) throws IOException, InterruptedException { @@ -120,16 +120,17 @@ static boolean crashTest(IRascalMonitor monitor, PrintWriter errorPrinter, Strin } } catch (Throwable e ) { - failed.set(true); - iFailed.set(true); err.println("❌ test fail :" + currentTest); err.println(e); + e.printStackTrace(err); + iFailed.set(true); } return null; }); } finally { // clean up memory var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestShared"); + if (memoryModule != null ) { var testRoot = memoryModule.getFrameVariable("testRoot"); try { URIResolverRegistry.getInstance().remove((ISourceLocation)testRoot.getValue(), true); @@ -138,6 +139,7 @@ static boolean crashTest(IRascalMonitor monitor, PrintWriter errorPrinter, Strin err.println("Failure to cleanup the cache"); } } + } } @@ -145,6 +147,7 @@ static boolean crashTest(IRascalMonitor monitor, PrintWriter errorPrinter, Strin errorPrinter.println("❌❌❌ Test run failed: " + name); errorPrinter.println("Job output:"); errorPrinter.println(output.toString()); + failed.set(true); return true; } return false; @@ -196,22 +199,24 @@ static boolean crashTestFreshEval(IRascalMonitor monitor, PrintWriter errorPrint } finally { // clean up memory var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestShared"); - var testRoot = memoryModule.getFrameVariable("testRoot"); - try { - URIResolverRegistry.getInstance().remove((ISourceLocation)testRoot.getValue(), true); - } - catch (Throwable e) { - err.println("Failure to cleanup the cache"); + if (memoryModule != null) { + var testRoot = memoryModule.getFrameVariable("testRoot"); + try { + URIResolverRegistry.getInstance().remove((ISourceLocation)testRoot.getValue(), true); + } + catch (Throwable e) { + err.println("Failure to cleanup the cache"); + } } } } } catch (Throwable e ) { - failed.set(true); iFailed.set(true); err.println("tests: " + Arrays.deepToString(tests.stream().map(AbstractFunction::getName).toArray())); err.println("❌ test fail :" + currentTest); err.println(e); + e.printStackTrace(err); } return null; }); @@ -221,6 +226,8 @@ static boolean crashTestFreshEval(IRascalMonitor monitor, PrintWriter errorPrint errorPrinter.println("❌❌❌ Test run failed: " + name); errorPrinter.println("Job output:"); errorPrinter.println(output.toString()); + errorPrinter.flush(); + failed.set(true); return true; } return false; From 1ba6f9c6f6c4d8865f4a284040e299c7a0a13f30 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 23 Mar 2026 16:09:55 +0100 Subject: [PATCH 19/19] Avoided extra module --- src/org/rascalmpl/compiler/BreakTest.java | 4 ++-- .../compiler/lang/rascalcore/check/TestConfigs.rsc | 8 +++++++- .../compiler/lang/rascalcore/check/TestShared.rsc | 8 -------- .../lang/rascalcore/check/tests/BinaryDependencyTests.rsc | 1 - .../lang/rascalcore/check/tests/StaticTestingUtils.rsc | 2 -- 5 files changed, 9 insertions(+), 14 deletions(-) delete mode 100644 src/org/rascalmpl/compiler/lang/rascalcore/check/TestShared.rsc diff --git a/src/org/rascalmpl/compiler/BreakTest.java b/src/org/rascalmpl/compiler/BreakTest.java index cdbcd34394d..112413ee547 100644 --- a/src/org/rascalmpl/compiler/BreakTest.java +++ b/src/org/rascalmpl/compiler/BreakTest.java @@ -129,7 +129,7 @@ static boolean crashTest(IRascalMonitor monitor, PrintWriter errorPrinter, Strin }); } finally { // clean up memory - var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestShared"); + var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestConfigs"); if (memoryModule != null ) { var testRoot = memoryModule.getFrameVariable("testRoot"); try { @@ -198,7 +198,7 @@ static boolean crashTestFreshEval(IRascalMonitor monitor, PrintWriter errorPrint } } finally { // clean up memory - var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestShared"); + var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestConfigs"); if (memoryModule != null) { var testRoot = memoryModule.getFrameVariable("testRoot"); try { diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc index 6e5c907c847..2cbb5ab0c8b 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc @@ -32,9 +32,12 @@ import lang::rascalcore::check::BasicRascalConfig; import lang::rascalcore::check::RascalConfig; import lang::rascalcore::check::ModuleLocations; -import lang::rascalcore::check::TestShared; // import lang::rascalcore::CompilerPathConfig; +import util::UUID; + + + // Duplicate in lang::rascalcore::compile::util::Names, factor out data PathConfig( loc generatedSources=|unknown:///|, @@ -65,6 +68,9 @@ loc TMP_COMPILED_RASCAL = |tmp:///compiled-rascal/|; // ---- PathConfigs for testing purposes -------------------------------------- +public loc testRoot = uuid()[scheme="memory"]; +public loc testModulesRoot = testRoot + "src"; + private int npc = 0; @synopsis{PathConfig for testing generated modules in |memory://test-modules/| in memory file system, not depending on any outside libraries.} @description{ diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestShared.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestShared.rsc deleted file mode 100644 index ac347670a50..00000000000 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestShared.rsc +++ /dev/null @@ -1,8 +0,0 @@ -module lang::rascalcore::check::TestShared - -import util::UUID; - - -public loc testRoot = uuid()[scheme="memory"]; -public loc testModulesRoot = testRoot + "src"; - diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc index 11ba7cf8f51..ecb76b91ded 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc @@ -46,7 +46,6 @@ import String; import lang::rascalcore::check::ModuleLocations; import util::FileSystem; import util::SemVer; -import lang::rascalcore::check::TestShared; import lang::rascalcore::check::tests::StaticTestingUtils; diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc index ec2b6ce300a..214ed2150a3 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc @@ -45,8 +45,6 @@ import ParseTree; import util::FileSystem; import lang::rascalcore::check::RascalConfig; -import lang::rascalcore::check::TestShared; - import lang::rascalcore::check::Checker; import lang::rascalcore::check::TestConfigs; import lang::rascal::\syntax::Rascal;