diff --git a/.vscode/launch.json b/.vscode/launch.json index fa9529bef36..789e4e10b33 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,14 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "type": "java", + "name": "BreakTest", + "request": "launch", + "mainClass": "org.rascalmpl.compiler.BreakTest", + "projectName": "rascal", + "console": "integratedTerminal" + }, { "type": "java", "name": "RascalCheck", diff --git a/src/org/rascalmpl/compiler/BreakTest.java b/src/org/rascalmpl/compiler/BreakTest.java new file mode 100644 index 00000000000..112413ee547 --- /dev/null +++ b/src/org/rascalmpl/compiler/BreakTest.java @@ -0,0 +1,235 @@ +package org.rascalmpl.compiler; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Reader; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import org.rascalmpl.debug.IRascalMonitor; +import org.rascalmpl.interpreter.Evaluator; +import org.rascalmpl.interpreter.result.AbstractFunction; +import org.rascalmpl.shell.RascalShell; +import org.rascalmpl.shell.ShellEvaluatorFactory; +import org.rascalmpl.test.infrastructure.RascalJUnitTestRunner; +import org.rascalmpl.uri.URIResolverRegistry; + +import io.usethesource.vallang.IBool; +import io.usethesource.vallang.ISourceLocation; + +public class BreakTest { + + // private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::ChangeScenarioTests"; + // private static final String BREAKING_TEST = "fixedErrorsDisappear2"; + private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::FunctionTCTests"; + private static final String BREAKING_TEST = "BoundOK1"; + + // notCompatibleAfterChangingFunctionArgument + // notCompatibleAfterChangingFunctionArgument + + private static final int PARALLEL_RUNS = 1; // set to 1 to avoid any multi-threading interactions, but it might take 20 rounds or something + private static final int TRIES = 1000 / PARALLEL_RUNS; + + public static void main(String[] args) throws IOException, InterruptedException { + RascalShell.setupJavaProcessForREPL(); + + var term = RascalShell.connectToTerminal(); + var monitor = IRascalMonitor.buildConsoleMonitor(term); + var error = monitor instanceof PrintWriter ? (PrintWriter) monitor : new PrintWriter(System.err, false); + AtomicBoolean failed = new AtomicBoolean(false); + try { + AtomicInteger done = new AtomicInteger(0); + for (int t = 0; t < PARALLEL_RUNS; t++) { + var name = "Thread " + (t + 1); + var tr = new Thread(() -> { + try { + if (crashTestFreshEval(monitor, error, name, failed)) { + failed.set(true); + error.flush(); + System.err.println("We got a failure, exiting now!"); + Thread.sleep(1000); + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + done.incrementAndGet(); + } + }); + tr.start(); + } + while (done.get() < PARALLEL_RUNS && !failed.get()) { + Thread.sleep(100); + } + } finally { + error.flush(); + error.close(); + } + if (failed.get()) { + System.out.flush(); + System.err.flush(); + System.exit(1); + } + } + + static boolean crashTest(IRascalMonitor monitor, PrintWriter errorPrinter, String name, AtomicBoolean failed) { + var output = new StringWriter(); + var iFailed = new AtomicBoolean(false); + try (var err = new PrintWriter(output, true); var out= new PrintWriter(output, false)) { + var projectRoot = RascalJUnitTestRunner.inferProjectRootFromClass(BreakTest.class); + var evaluator = ShellEvaluatorFactory.getDefaultEvaluatorForLocation(projectRoot, Reader.nullReader(), err, out, monitor, "$test-"+name+"$"); + evaluator.getConfiguration().setErrors(true); + // make sure we're writing to the outputs + evaluator.overwritePrintWriter(out, err); + + evaluator.doImport(monitor, BREAKING_MODULE); + try { + monitor.job(name, TRIES, (jobname, step) -> { + String currentTest = ""; + var tests = evaluator.getHeap().getModule(BREAKING_MODULE).getTests(); + try { + tests = tests.stream() + .filter(t -> !t.hasTag("ignore")) + .collect(Collectors.toList()); + + monitor.jobTodo(jobname, tests.size() * TRIES); + + for (int i = 0; i < TRIES; i++) { + if (failed.get()) { + return false; + } + + Collections.shuffle(tests); + + monitor.jobStep(jobname, "Running: try " + (i + 1)); + for (var t: tests) { + currentTest = t.getName(); + monitor.jobStep(jobname, "Running: try " + (i + 1) + " => " + currentTest); + var result = t.call(); + if (!((IBool)result).getValue()) { + throw new RuntimeException("Test " + currentTest +" returned false"); + } + } + } + + } catch (Throwable e ) { + err.println("❌ test fail :" + currentTest); + err.println(e); + e.printStackTrace(err); + iFailed.set(true); + } + return null; + }); + } finally { + // clean up memory + var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestConfigs"); + if (memoryModule != null ) { + var testRoot = memoryModule.getFrameVariable("testRoot"); + try { + URIResolverRegistry.getInstance().remove((ISourceLocation)testRoot.getValue(), true); + } + catch (Throwable e) { + err.println("Failure to cleanup the cache"); + } + } + } + + } + + if (iFailed.get()) { + errorPrinter.println("❌❌❌ Test run failed: " + name); + errorPrinter.println("Job output:"); + errorPrinter.println(output.toString()); + failed.set(true); + return true; + } + return false; + } + + static boolean crashTestFreshEval(IRascalMonitor monitor, PrintWriter errorPrinter, String name, AtomicBoolean failed) { + var output = new StringWriter(); + var iFailed = new AtomicBoolean(false); + try (var err = new PrintWriter(output, true); var out= new PrintWriter(output, false)) { + var projectRoot = RascalJUnitTestRunner.inferProjectRootFromClass(BreakTest.class); + monitor.job(name, TRIES * 2, (jobname, step) -> { + String currentTest = ""; + var tests = Collections.emptyList(); + try { + for (int i = 0; i < TRIES; i++) { + if (failed.get()) { + return false; + } + monitor.jobStep(jobname, "Try " + (i + 1)); + var evaluator = ShellEvaluatorFactory.getDefaultEvaluatorForLocation(projectRoot, Reader.nullReader(), err, out, monitor, "$test-"+name+"$"); + try { + + evaluator.getConfiguration().setErrors(true); + // make sure we're writing to the outputs + evaluator.overwritePrintWriter(out, err); + + evaluator.doImport(monitor, BREAKING_MODULE); + + tests = evaluator.getHeap().getModule(BREAKING_MODULE).getTests() + .stream() + .filter(t -> !t.hasTag("ignore")) + .collect(Collectors.toList()); + + if (i == 0) { + monitor.jobTodo(jobname, tests.size() * TRIES); + } + + Collections.shuffle(tests); + + monitor.jobStep(jobname, "Running: try " + (i + 1)); + for (var t: tests) { + currentTest = t.getName(); + monitor.jobStep(jobname, "Running: try " + (i + 1) + " => " + currentTest); + var result = t.call(); + if (!((IBool)result).getValue()) { + throw new RuntimeException("Test " + currentTest +" returned false"); + } + } + } finally { + // clean up memory + var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestConfigs"); + if (memoryModule != null) { + var testRoot = memoryModule.getFrameVariable("testRoot"); + try { + URIResolverRegistry.getInstance().remove((ISourceLocation)testRoot.getValue(), true); + } + catch (Throwable e) { + err.println("Failure to cleanup the cache"); + } + } + } + } + + } catch (Throwable e ) { + iFailed.set(true); + err.println("tests: " + Arrays.deepToString(tests.stream().map(AbstractFunction::getName).toArray())); + err.println("❌ test fail :" + currentTest); + err.println(e); + e.printStackTrace(err); + } + return null; + }); + } + + if (iFailed.get()) { + errorPrinter.println("❌❌❌ Test run failed: " + name); + errorPrinter.println("Job output:"); + errorPrinter.println(output.toString()); + errorPrinter.flush(); + failed.set(true); + return true; + } + return false; + } +} diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc index 55fadfad48e..e96b48f02ab 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc @@ -34,6 +34,10 @@ import lang::rascalcore::check::RascalConfig; import lang::rascalcore::check::ModuleLocations; // import lang::rascalcore::CompilerPathConfig; +import util::UUID; + + + // Duplicate in lang::rascalcore::compile::util::Names, factor out data PathConfig( loc generatedSources=|unknown:///|, @@ -64,6 +68,9 @@ loc TMP_COMPILED_RASCAL = |tmp:///compiled-rascal/|; // ---- PathConfigs for testing purposes -------------------------------------- +public loc testRoot = |memory://6e17d46a-06e9-42aa-bd98-182ca2dbd8d3/|;// uuid()[scheme="memory"]; +public loc testModulesRoot = testRoot + "src"; + private int npc = 0; @synopsis{PathConfig for testing generated modules in |memory://test-modules/| in memory file system, not depending on any outside libraries.} @description{ @@ -74,10 +81,10 @@ public PathConfig getDefaultTestingPathConfig() { npc += 1; snpc = ""; return pathConfig( - srcs = [ |memory:///test-modules/|, |std:///| ], - bin = |memory:///test-modules/rascal-tests-bin-|, - generatedSources = |memory:///test-modules/generated-test-sources-|, - generatedResources = |memory:///test-modules/generated-test-resources-|, + srcs = [ testModulesRoot, |std:///| ], + bin = testRoot + "rascal-tests-bin-", + generatedSources = testRoot + "generated-test-sources-", + generatedResources = testRoot + "generated-test-resources-", libs = [ ] ); } @@ -92,10 +99,10 @@ public PathConfig getReleasedStandardLibraryTestingPathConfig() { npc += 1; snpc = ""; return pathConfig( - srcs = [ |memory:///test-modules/| ], - bin = |memory:///test-modules/rascal-tests-bin-|, - generatedSources = |memory:///test-modules/generated-test-sources-|, - generatedResources = |memory:///test-modules/generated-test-resources-|, + srcs = [ testModulesRoot ], + bin = testRoot + "rascal-tests-bin-", + generatedSources = testRoot + "generated-test-sources-", + generatedResources = testRoot + "generated-test-resources-", libs = [ |lib://rascal| ] ); } @@ -108,7 +115,7 @@ public PathConfig getRascalProjectTestingPathConfig() { snpc = ""; return pathConfig( srcs = [|project://rascal/src/org/rascalmpl/library|], - bin = |memory:///test-modules/rascal-lib-bin-|, + bin = testRoot + "rascal-lib-bin-", libs = [] ); } diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc index 1cee586e9f1..ecb76b91ded 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc @@ -58,10 +58,10 @@ data PathConfig(loc generatedResources=|unknown:///|, loc generatedSources=|unkn data Project = project(str name, map[str moduleName, str moduleText] modules, PathConfig pcfg); -void clearMemory() { remove(|memory:///|, recursive = true); } +void clearMemory() { remove(testRoot, recursive = true); } loc projectDir(str pname) - = |memory:///|; + = testRoot + pname; loc src(str pname) = projectDir(pname) + "src/"; diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BreakingTest.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BreakingTest.rsc new file mode 100644 index 00000000000..de2a11fc068 --- /dev/null +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BreakingTest.rsc @@ -0,0 +1,86 @@ +module lang::rascalcore::check::tests::BreakingTest + +import IO; +import String; +import Location; +import Message; +import Set; +import util::Reflective; +import ParseTree; +import lang::rascalcore::check::RascalConfig; + +import lang::rascalcore::check::Checker; +import lang::rascal::\syntax::Rascal; + + +// this uuid matters +loc root = |memory://6e17d46a-06e9-42aa-bd98-182ca2dbd8d3/|; +PathConfig pcfg = pathConfig( + srcs = [root + "src"], + bin = root + "bin", + libs = [] +); +// this name matters +str moduleName = "TestModule612d1"; + +loc writeModule() { + loc moduleLoc = pcfg.srcs[0] + ".rsc"; + // the spaces before &T seems to matter? + writeFile(moduleLoc, + "module + ' &T \<: int f(&T \<: num _) = 42; + '" + ); + return moduleLoc; +} + + +set[Message] getErrorMessages(ModuleStatus r) + = { m | m <- getAllMessages(r), m is error }; + +set[Message] getWarningMessages(ModuleStatus r) + = { m | m <- getAllMessages(r), m is warning }; + +set[Message] getAllMessages(ModuleStatus r) + = { m | mname <- r.messages, m <- r.messages[mname] }; + +bool typecheckModule(loc m) { + status = rascalTModelForLocs([m], rascalCompilerConfig(pcfg)[infoModuleChecked=true][verbose=true], dummy_compile1); + println("All messages:"); + iprintln(getAllMessages(status)); + if (e <- getErrorMessages(status)) { + println("Errors typechecking: "); + iprintln(getErrorMessages(status)); + println("❌ Typecheck failed"); + return false; + } + else { + println("✅ Typecheck success"); + return true; + } +} + +void findCollission(loc l) { + m = parseModule(l); + locs = [ t.src | /Tree t := m, t.src?]; + println("Found locs"); + locsSet = {*locs}; + println("Became: locs when putting in set"); + for (l <- locs) { + bool found = false; + for (l2 <- locsSet, "" == "") { + found = true; + } + if (!found) { + println("❌ got dropped from set"); + } + } +} + + +void main() { + remove(root, recursive = true); + l = writeModule(); + typecheckModule(l); + findCollission(l); +} \ No newline at end of file diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc index cef2ace5073..58b812dcc6a 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc @@ -882,4 +882,4 @@ void allBenchmarks(){ mediumBenchmarkRechecking(); //largeBenchmarkRechecking(); println("Total time: <(cpuTime() - beginTime)/1000000> ms"); -} \ No newline at end of file +} diff --git a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc index 8393190b56a..d44dc0dd193 100644 --- a/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc +++ b/src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc @@ -75,7 +75,7 @@ tuple[str,str] extractModuleNameAndBody(str moduleText){ loc composeModule(str stmts){ return writeModule( - "module TestModule + "module TestModule 'value main(){ ' \n ' return true; @@ -83,14 +83,14 @@ loc composeModule(str stmts){ } void clearMemory() { - remove(|memory:///test-modules/| recursive = true); + remove(testRoot, recursive = true); } str cleanName(str name) = name[0] == "\\" ? name[1..] : name; loc writeModule(str moduleText){ = extractModuleNameAndBody(moduleText); - mloc = |memory:///test-modules/.rsc|; + mloc = testModulesRoot + ".rsc"; writeFile(mloc, moduleText); return mloc; } @@ -101,7 +101,7 @@ list[loc] writeModules(str modules...) void removeModule(str mname){ pcfg = getDefaultTestingPathConfig(); name = cleanName(mname); - remove(|memory:///test-modules/.rsc|); + remove(testModulesRoot + ".rsc"); remove(pcfg.generatedResources + ".tpl"); } @@ -109,7 +109,7 @@ void printModules(){ println("\<\<\<\<"); for(f <- find(testRoot, "rsc")){ println(" : - '"); + '"); } for(f <- find(testRoot, "tpl")){ println(": "); @@ -117,7 +117,6 @@ void printModules(){ println("\>\>\>\>"); } - set[Message] getErrorMessages(ModuleStatus r) = { m | m <- getAllMessages(r), m is error }; @@ -573,4 +572,4 @@ bool expectReChecksWithErrors(loc mloc, list[str] moduleNames, PathConfig pathCo msgs = [ "Checked " | nm <- moduleNames ]; mlocs = mloc + [ getModuleLocation(mn, pathConfig) | mn <- moduleNames ]; return checkModuleAndFilter(mlocs, msgs, matchAll=true, errorsAllowed=true, pathConfig=pathConfig); -} \ No newline at end of file +} diff --git a/test/org/rascalmpl/compiler/lang/rascalcore/check/tests/RunTests.java b/test/org/rascalmpl/compiler/lang/rascalcore/check/tests/RunTests.java index 05cf4b7b415..26e6a6591bf 100644 --- a/test/org/rascalmpl/compiler/lang/rascalcore/check/tests/RunTests.java +++ b/test/org/rascalmpl/compiler/lang/rascalcore/check/tests/RunTests.java @@ -27,11 +27,13 @@ package org.rascalmpl.compiler.lang.rascalcore.check.tests; import org.junit.runner.RunWith; +import org.rascalmpl.test.infrastructure.RascalJUnitParallelRecursiveTestRunner; import org.rascalmpl.test.infrastructure.RascalJUnitTestPrefix; import org.rascalmpl.test.infrastructure.RascalJUnitTestRunner; +import org.rascalmpl.test.infrastructure.RecursiveRascalParallelTest; -@RunWith(RascalJUnitTestRunner.class) -@RascalJUnitTestPrefix("lang::rascalcore::check::tests") +@RunWith(RascalJUnitParallelRecursiveTestRunner.class) +@RecursiveRascalParallelTest("lang::rascalcore::check::tests") public class RunTests { }