From 70ea99d1d96f6734e6cb109b87dbd9781b386e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Schr=C3=B6der?= Date: Thu, 4 Jun 2026 14:06:12 +0200 Subject: [PATCH 1/2] Generator class and test with rml-core --- pom.xml | 420 +++++++++++++------------- src/main/java/burp/Generator.java | 200 ++++++++++++ src/test/java/burp/TestGenerator.java | 165 ++++++++++ 3 files changed, 578 insertions(+), 207 deletions(-) create mode 100644 src/main/java/burp/Generator.java create mode 100644 src/test/java/burp/TestGenerator.java diff --git a/pom.xml b/pom.xml index 7eceab29..69c8e87a 100644 --- a/pom.xml +++ b/pom.xml @@ -1,220 +1,226 @@ - 4.0.0 - BURP - BURP - 0.1.2-SNAPSHOT - Basic and Unassuming RML Processor + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + BURP + BURP + 0.1.2-SNAPSHOT + Basic and Unassuming RML Processor - - 17 - 17 - UTF-8 - + + 17 + 17 + UTF-8 + - - - jitpack.io - https://jitpack.io - - + + + jitpack.io + https://jitpack.io + + - - src/main/java - - - src/main/resources - - **/*.ttl - - - + + src/main/java + + + src/main/resources + + **/*.ttl + + + - - - maven-compiler-plugin - 3.8.1 - - 17 - - + + + maven-compiler-plugin + 3.8.1 + + 17 + + - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + - - org.apache.maven.plugins - maven-jar-plugin - 3.3.0 - - - - true - dependency/ - burp.Main - - - - + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + true + dependency/ + burp.Main + + + + - - org.apache.maven.plugins - maven-shade-plugin - 2.4.3 - - - package - - shade - - - true - - - burp.Main - - - - - false - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - burp - - - - - - + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + + package + + shade + + + true + + + burp.Main + + + + + false + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + burp + + + + + + - - - info.picocli - picocli - 4.7.5 - - - org.apache.jena - jena-arq - 4.10.0 - - - com.opencsv - opencsv - 5.9 - - - com.jayway.jsonpath - json-path - 2.9.0 - - - net.minidev - json-smart - 2.5.0 - - - org.apache.jena - jena-shacl - 4.10.0 - - - mysql - mysql-connector-java - 8.0.33 - - - org.postgresql - postgresql - 42.7.2 - - - com.microsoft.sqlserver - mssql-jdbc - 12.6.1.jre11 - - - org.xerial - sqlite-jdbc - 3.46.0.0 - - - org.apache.commons - commons-text - 1.11.0 - + + + info.picocli + picocli + 4.7.5 + + + org.apache.jena + jena-arq + 4.10.0 + + + com.opencsv + opencsv + 5.9 + + + com.jayway.jsonpath + json-path + 2.9.0 + + + net.minidev + json-smart + 2.5.0 + + + org.apache.jena + jena-shacl + 4.10.0 + + + mysql + mysql-connector-java + 8.0.33 + + + org.postgresql + postgresql + 42.7.2 + + + com.microsoft.sqlserver + mssql-jdbc + 12.6.1.jre11 + + + org.xerial + sqlite-jdbc + 3.46.0.0 + + + org.apache.commons + commons-text + 1.11.0 + - - - org.junit.jupiter - junit-jupiter-engine - 5.10.2 - test - - - org.junit.jupiter - junit-jupiter-api - 5.10.2 - test - - - org.testcontainers - junit-jupiter - 1.19.5 - test - - - org.testcontainers - postgresql - 1.19.5 - test - - - org.testcontainers - mysql - 1.19.5 - test - - - org.apache.jena - jena-fuseki-main - 4.10.0 - test - + + + org.junit.jupiter + junit-jupiter-engine + 5.10.2 + test + + + org.junit.jupiter + junit-jupiter-api + 5.10.2 + test + + + org.testcontainers + junit-jupiter + 1.19.5 + test + + + org.junit.jupiter + junit-jupiter-params + 5.10.2 + test + + + org.testcontainers + postgresql + 1.19.5 + test + + + org.testcontainers + mysql + 1.19.5 + test + + + org.apache.jena + jena-fuseki-main + 4.10.0 + test + - - org.tukaani - xz - 1.9 - - - com.univocity - univocity-parsers - 2.9.1 - - - com.github.candil-data-fabric - JNC - xpath-namespace-SNAPSHOT - - + + org.tukaani + xz + 1.9 + + + com.univocity + univocity-parsers + 2.9.1 + + + com.github.candil-data-fabric + JNC + xpath-namespace-SNAPSHOT + + diff --git a/src/main/java/burp/Generator.java b/src/main/java/burp/Generator.java new file mode 100644 index 00000000..e57c2bad --- /dev/null +++ b/src/main/java/burp/Generator.java @@ -0,0 +1,200 @@ +package burp; + +import burp.model.GraphMap; +import burp.model.Iteration; +import burp.model.ObjectMap; +import burp.model.PredicateMap; +import burp.model.PredicateObjectMap; +import burp.model.ReferencingObjectMap; +import burp.model.SubjectMap; +import burp.model.TriplesMap; +import burp.model.gathermaputil.SubGraph; +import burp.vocabularies.RML; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.function.Consumer; +import org.apache.jena.graph.Node; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.sparql.core.Quad; +import org.apache.jena.vocabulary.RDF; + +/** + * Generates quadruples by executing supplied RML triples maps. + */ +public class Generator { + + private final List def = List.of(RML.defaultGraph); + + /** + * Executes the supplied RML triples maps and streams the generated RDF quads to the provided consumer. + *

+ * No dataset is materialized in memory. Each generated quad is emitted immediately by invoking {@link Consumer#accept(Object)} on the supplied consumer. + * The order of emitted quads follows the order in which triples maps, logical source iterations, subject maps, and predicate-object maps are processed by + * the generator. A {@code Consumer} is intended for side-effecting operations such as writing, indexing, or collecting generated data. + * + * @param triplesmaps the triples maps to execute + * @param baseIRI base IRI used to resolve relative IRIs produced by the mapping + * @param consumer callback invoked once for every generated quad + */ + public void generate(List triplesmaps, + String baseIRI, + Consumer consumer) { + + for (TriplesMap tm : triplesmaps) { + + SubjectMap sm = tm.subjectMap; + List sgm = sm.graphMaps; + + Iterator iter = tm.logicalSource.iterator(); + while (iter.hasNext()) { + + Iteration i = iter.next(); + + // subject graphs + List sgs = sgm.isEmpty() + ? def + : new ArrayList<>(); + + for (GraphMap gm : sgm) { + sgs.addAll(gm.generateTerms(i, baseIRI)); + } + + // subjects + List subjects = new ArrayList<>(); + + if (!sm.isGatherMap()) { + subjects.addAll(sm.generateTerms(i, baseIRI)); + } else { + for (SubGraph subgraph : sm.generateGatherMapGraphs(i, baseIRI)) { + subjects.add(subgraph.node); + emitSubGraph(subgraph, sgs, consumer); + } + } + + // rdf:type from subject map + emitSubjectTypes(sm.classes, subjects, sgs, consumer); + + // predicate-object maps + for (PredicateObjectMap pom : tm.predicateObjectMaps) { + + List pogs = new ArrayList<>(); + for (GraphMap gm : pom.graphMaps) { + pogs.addAll(gm.generateTerms(i, baseIRI)); + } + + List graphs = def; + + if (!sgm.isEmpty() || !pogs.isEmpty()) { + pogs.addAll(!sgm.isEmpty() ? sgs : new ArrayList<>()); + graphs = pogs; + } + + List predicates = new ArrayList<>(); + for (PredicateMap pm : pom.predicateMaps) { + predicates.addAll(pm.generateTerms(i, baseIRI)); + } + + List objects = new ArrayList<>(); + + for (ObjectMap om : pom.objectMaps) { + if (!om.isGatherMap()) { + objects.addAll(om.generateTerms(i, baseIRI)); + } else { + for (SubGraph subgraph : om.generateGatherMapGraphs(i, baseIRI)) { + objects.add(subgraph.node); + emitSubGraph(subgraph, graphs, consumer); + } + } + } + + for (ReferencingObjectMap rom : pom.refObjectMaps) { + objects.addAll(rom.generateTerms(i, baseIRI)); + } + + emitTriples(subjects, predicates, objects, graphs, consumer); + } + } + } + } + + /* ---------------- Quad emission helpers ---------------- */ + private void emitTriples(List subjects, + List predicates, + List objects, + List graphs, + Consumer consumer) { + + for (RDFNode s : subjects) { + Node sn = s.asNode(); + + for (RDFNode p : predicates) { + Node pn = p.asNode(); + + for (RDFNode o : objects) { + Node on = o.asNode(); + + for (RDFNode g : graphs) { + Node gn = toGraphNode(g); + consumer.accept(Quad.create(gn, sn, pn, on)); + } + } + } + } + } + + private void emitSubjectTypes(List classes, + List subjects, + List graphs, + Consumer consumer) { + + Node rdfType = RDF.type.asNode(); + + for (RDFNode s : subjects) { + Node sn = s.asNode(); + + for (Resource c : classes) { + Node cn = c.asNode(); + + for (RDFNode g : graphs) { + Node gn = toGraphNode(g); + consumer.accept(Quad.create(gn, sn, rdfType, cn)); + } + } + } + } + + /* ---------------- SubGraph handling ---------------- */ + private void emitSubGraph(SubGraph subgraph, + List graphs, + Consumer consumer) { + + Node gn = null; + + for (RDFNode g : graphs) { + gn = toGraphNode(g); + + Node finalGn = gn; + + // emit all triples from subgraph.model into quads + subgraph.model.listStatements().forEachRemaining(st -> { + consumer.accept(Quad.create( + finalGn, + st.getSubject().asNode(), + st.getPredicate().asNode(), + st.getObject().asNode() + )); + }); + } + } + + /* ---------------- graph conversion ---------------- */ + private Node toGraphNode(RDFNode g) { + if (g.equals(RML.defaultGraph)) { + return Quad.defaultGraphIRI; + } + return g.asNode(); + } + +} diff --git a/src/test/java/burp/TestGenerator.java b/src/test/java/burp/TestGenerator.java new file mode 100644 index 00000000..bfd8cd86 --- /dev/null +++ b/src/test/java/burp/TestGenerator.java @@ -0,0 +1,165 @@ +package burp; + +import burp.model.TriplesMap; +import burp.parse.Parse; +import java.io.File; +import java.util.List; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.DatasetGraphFactory; +import org.junit.jupiter.api.Assertions; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * + */ +public class TestGenerator { + + private static String base = "./src/test/resources/rml-core/"; + + @ParameterizedTest + @ValueSource(strings = { + "RMLTC0000-CSV", + "RMLTC0000-JSON", + "RMLTC0000-XML", + "RMLTC0001a-CSV", + "RMLTC0001a-JSON", + "RMLTC0001a-XML", + "RMLTC0001b-CSV", + "RMLTC0001b-JSON", + "RMLTC0001b-XML", + "RMLTC0002a-CSV", + "RMLTC0002a-JSON", + "RMLTC0002a-XML", + "RMLTC0002b-CSV", + "RMLTC0002b-JSON", + "RMLTC0002b-XML", + "RMLTC0003c-CSV", + "RMLTC0003c-JSON", + "RMLTC0003c-XML", + "RMLTC0004a-CSV", + "RMLTC0004a-JSON", + "RMLTC0004a-XML", + "RMLTC0005a-CSV", + "RMLTC0005a-JSON", + "RMLTC0005a-XML", + "RMLTC0006a-CSV", + "RMLTC0006a-JSON", + "RMLTC0006a-XML", + "RMLTC0007a-CSV", + "RMLTC0007a-JSON", + "RMLTC0007a-XML", + "RMLTC0007b-CSV", + "RMLTC0007b-JSON", + "RMLTC0007b-XML", + "RMLTC0007c-CSV", + "RMLTC0007c-JSON", + "RMLTC0007c-XML", + "RMLTC0007d-CSV", + "RMLTC0007d-JSON", + "RMLTC0007d-XML", + "RMLTC0007e-CSV", + "RMLTC0007e-JSON", + "RMLTC0007e-XML", + "RMLTC0007f-CSV", + "RMLTC0007f-JSON", + "RMLTC0007f-XML", + "RMLTC0007g-CSV", + "RMLTC0007g-JSON", + "RMLTC0007g-XML", + "RMLTC0008a-CSV", + "RMLTC0008a-JSON", + "RMLTC0008a-XML", + "RMLTC0008b-CSV", + "RMLTC0008b-JSON", + "RMLTC0008b-XML", + "RMLTC0008c-CSV", + "RMLTC0008c-JSON", + "RMLTC0008c-XML", + "RMLTC0009a-CSV", + "RMLTC0009a-JSON", + "RMLTC0009a-XML", + "RMLTC0009b-CSV", + "RMLTC0009b-JSON", + "RMLTC0009b-XML", + "RMLTC0010a-CSV", + "RMLTC0010a-JSON", + "RMLTC0010b-CSV", + "RMLTC0010b-JSON", + "RMLTC0010b-XML", + "RMLTC0010c-CSV", + "RMLTC0010c-JSON", + "RMLTC0010c-XML", + "RMLTC0011b-CSV", + "RMLTC0011b-JSON", + "RMLTC0011b-XML", + "RMLTC0012a-CSV", + "RMLTC0012a-JSON", + "RMLTC0012a-XML", + "RMLTC0012b-CSV", + "RMLTC0012b-JSON", + "RMLTC0012b-XML", + "RMLTC0013a-JSON", + "RMLTC0015a-CSV", + "RMLTC0015a-JSON", + "RMLTC0015a-XML", + "RMLTC0019a-CSV", + "RMLTC0019a-JSON", + "RMLTC0019a-XML", + "RMLTC0020a-CSV", + "RMLTC0020a-JSON", + "RMLTC0020a-XML", + "RMLTC0021a-CSV", + "RMLTC0021a-JSON", + "RMLTC0021a-XML", + //"RMLTC0022a-CSV", + //"RMLTC0022b-CSV", + //"RMLTC0022c-CSV", + "RMLTC0023a-XML", + //"RMLTC0026a-JSON", + "RMLTC0027-XML" + }) + public void rmlTest(String folder) throws Exception { + String m = new File(base + folder, "mapping.ttl").getAbsolutePath(); + + // Parse the mapping file + List triplesmaps = Parse.parseMappingFile(m); + + DatasetGraph actual = DatasetGraphFactory.create(); + + String baseIRI = "http://example.com/base/"; + + Generator generator = new Generator(); + + try { + generator.generate(triplesmaps, baseIRI, quad -> { + actual.add(quad); + }); + } catch(Exception e) { + System.out.println(folder); + Assertions.fail(e); + } + + String o = new File(base + folder, "output.nq").getAbsolutePath(); + Model expected = RDFDataMgr.loadModel(o); + + boolean b = actual.getDefaultGraph().isIsomorphicWith(expected.getGraph()); + + if(!b) { + System.out.println(folder); + System.out.println("Expected:"); + expected.write(System.out, "Turtle"); + System.out.println("---"); + System.out.println("Actual:"); + RDFDataMgr.write(System.out, actual, Lang.TRIG); + System.out.println("==============="); + } + + assertTrue(b); + } + +} From 768c727e91d1543c7ccc2c0485dce117151a3ba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Schr=C3=B6der?= Date: Sat, 6 Jun 2026 09:21:21 +0200 Subject: [PATCH 2/2] default graph parameter --- src/main/java/burp/Generator.java | 23 ++++++++++++++--------- src/test/java/burp/TestGenerator.java | 3 ++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/main/java/burp/Generator.java b/src/main/java/burp/Generator.java index e57c2bad..d1900afa 100644 --- a/src/main/java/burp/Generator.java +++ b/src/main/java/burp/Generator.java @@ -36,10 +36,12 @@ public class Generator { * * @param triplesmaps the triples maps to execute * @param baseIRI base IRI used to resolve relative IRIs produced by the mapping + * @param defaultGraph the default graph to be used * @param consumer callback invoked once for every generated quad */ public void generate(List triplesmaps, String baseIRI, + Node defaultGraph, Consumer consumer) { for (TriplesMap tm : triplesmaps) { @@ -69,12 +71,12 @@ public void generate(List triplesmaps, } else { for (SubGraph subgraph : sm.generateGatherMapGraphs(i, baseIRI)) { subjects.add(subgraph.node); - emitSubGraph(subgraph, sgs, consumer); + emitSubGraph(subgraph, sgs, defaultGraph, consumer); } } // rdf:type from subject map - emitSubjectTypes(sm.classes, subjects, sgs, consumer); + emitSubjectTypes(sm.classes, subjects, sgs, defaultGraph, consumer); // predicate-object maps for (PredicateObjectMap pom : tm.predicateObjectMaps) { @@ -104,7 +106,7 @@ public void generate(List triplesmaps, } else { for (SubGraph subgraph : om.generateGatherMapGraphs(i, baseIRI)) { objects.add(subgraph.node); - emitSubGraph(subgraph, graphs, consumer); + emitSubGraph(subgraph, graphs, defaultGraph, consumer); } } } @@ -113,7 +115,7 @@ public void generate(List triplesmaps, objects.addAll(rom.generateTerms(i, baseIRI)); } - emitTriples(subjects, predicates, objects, graphs, consumer); + emitTriples(subjects, predicates, objects, graphs, defaultGraph, consumer); } } } @@ -124,6 +126,7 @@ private void emitTriples(List subjects, List predicates, List objects, List graphs, + Node defaultGraph, Consumer consumer) { for (RDFNode s : subjects) { @@ -136,7 +139,7 @@ private void emitTriples(List subjects, Node on = o.asNode(); for (RDFNode g : graphs) { - Node gn = toGraphNode(g); + Node gn = toGraphNode(g, defaultGraph); consumer.accept(Quad.create(gn, sn, pn, on)); } } @@ -147,6 +150,7 @@ private void emitTriples(List subjects, private void emitSubjectTypes(List classes, List subjects, List graphs, + Node defaultGraph, Consumer consumer) { Node rdfType = RDF.type.asNode(); @@ -158,7 +162,7 @@ private void emitSubjectTypes(List classes, Node cn = c.asNode(); for (RDFNode g : graphs) { - Node gn = toGraphNode(g); + Node gn = toGraphNode(g, defaultGraph); consumer.accept(Quad.create(gn, sn, rdfType, cn)); } } @@ -168,12 +172,13 @@ private void emitSubjectTypes(List classes, /* ---------------- SubGraph handling ---------------- */ private void emitSubGraph(SubGraph subgraph, List graphs, + Node defaultGraph, Consumer consumer) { Node gn = null; for (RDFNode g : graphs) { - gn = toGraphNode(g); + gn = toGraphNode(g, defaultGraph); Node finalGn = gn; @@ -190,9 +195,9 @@ private void emitSubGraph(SubGraph subgraph, } /* ---------------- graph conversion ---------------- */ - private Node toGraphNode(RDFNode g) { + private Node toGraphNode(RDFNode g, Node defaultGraph) { if (g.equals(RML.defaultGraph)) { - return Quad.defaultGraphIRI; + return defaultGraph; } return g.asNode(); } diff --git a/src/test/java/burp/TestGenerator.java b/src/test/java/burp/TestGenerator.java index bfd8cd86..2eed972b 100644 --- a/src/test/java/burp/TestGenerator.java +++ b/src/test/java/burp/TestGenerator.java @@ -9,6 +9,7 @@ import org.apache.jena.riot.RDFDataMgr; import org.apache.jena.sparql.core.DatasetGraph; import org.apache.jena.sparql.core.DatasetGraphFactory; +import org.apache.jena.sparql.core.Quad; import org.junit.jupiter.api.Assertions; import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.params.ParameterizedTest; @@ -136,7 +137,7 @@ public void rmlTest(String folder) throws Exception { Generator generator = new Generator(); try { - generator.generate(triplesmaps, baseIRI, quad -> { + generator.generate(triplesmaps, baseIRI, Quad.defaultGraphIRI, quad -> { actual.add(quad); }); } catch(Exception e) {