From f23a0da3ce776dfea928a676a410a2609fb0a93f Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Wed, 29 Apr 2026 06:34:59 +0200 Subject: [PATCH 1/2] feat(#37): support importing multiple PBF files, update metadata structure and import logic --- .../paikka/PaikkaApplication.java | 48 +++++++++++++------ .../paikka/service/MetadataService.java | 2 +- .../paikka/service/PaikkaMetadata.java | 4 +- .../service/importer/ImportService.java | 29 ++++++----- 4 files changed, 55 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java b/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java index 6cb1d13..d6f27ef 100644 --- a/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java +++ b/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java @@ -24,6 +24,8 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import java.util.*; + @SpringBootApplication public class PaikkaApplication implements CommandLineRunner { @@ -70,33 +72,49 @@ private void printApiInfo() { logger.info(" curl 'http://localhost:8080/api/v1/geometry/12345'"); logger.info(""); } - + @Override public void run(String... args) throws Exception { - // Only run import logic if --import flag is present boolean isImportMode = false; - String pbfFile = null; + List pbfFiles = new ArrayList<>(); String dataDir = "./data"; - + Set usedArgIndices = new HashSet<>(); + + // Process flags and their values first for (int i = 0; i < args.length; i++) { - if ("--import".equals(args[i])) { + String arg = args[i]; + if ("--import".equals(arg)) { isImportMode = true; - } else if ("--pbf-file".equals(args[i]) && i + 1 < args.length) { - pbfFile = args[i + 1]; - } else if ("--data-dir".equals(args[i]) && i + 1 < args.length) { - dataDir = args[i + 1]; + } else if ("--pbf-file".equals(arg)) { + if (i + 1 >= args.length) { logger.error("Missing --pbf-file value"); System.exit(1); } + String value = args[ i + 1]; + usedArgIndices.add(i + 1); + // Split comma-separated values, add non-empty trimmed paths + Arrays.stream(value.split(",")).map(String::trim).filter(s -> !s.isEmpty()).forEach(pbfFiles::add); + i++; // Skip flag value + } else if ("--data-dir".equals(arg)) { + if (i + 1 >= args.length) { logger.error("Missing --data-dir value"); System.exit(1); } + dataDir = args[ i + 1]; + usedArgIndices.add(i + 1); + i++; // Skip flag value } } - + + // Collect trailing positional args (not flags/flag values) as PBF files + for (int i = 0; i < args.length; i++) { + if (usedArgIndices.contains(i)) continue; + String arg = args[i]; + if (arg.startsWith("--")) continue; // Skip unrecognized flags + if (isImportMode) pbfFiles.add(arg.trim()); + } + if (isImportMode) { - if (pbfFile == null) { - logger.error("Import mode requires --pbf-file argument"); + if (pbfFiles.isEmpty()) { + logger.error("Import mode requires at least one PBF file (use --pbf-file or trailing positional args)"); System.exit(1); } - - try { - importService.importData(pbfFile, dataDir); + importService.importData(pbfFiles, dataDir); System.exit(0); } catch (Exception e) { logger.error("Import failed", e); diff --git a/src/main/java/com/dedicatedcode/paikka/service/MetadataService.java b/src/main/java/com/dedicatedcode/paikka/service/MetadataService.java index 746418f..309abdd 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/MetadataService.java +++ b/src/main/java/com/dedicatedcode/paikka/service/MetadataService.java @@ -110,7 +110,7 @@ public Map getMetadata() { Map metadataMap = new HashMap<>(); metadataMap.put("importTimestamp", metadata.importTimestamp()); metadataMap.put("dataVersion", metadata.dataVersion()); - metadataMap.put("file", metadata.file()); + metadataMap.put("files", metadata.files()); metadataMap.put("gridLevel", metadata.gridLevel()); metadataMap.put("paikkaVersion", metadata.paikkaVersion()); return Collections.unmodifiableMap(metadataMap); diff --git a/src/main/java/com/dedicatedcode/paikka/service/PaikkaMetadata.java b/src/main/java/com/dedicatedcode/paikka/service/PaikkaMetadata.java index 37cd74b..69d4d8f 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/PaikkaMetadata.java +++ b/src/main/java/com/dedicatedcode/paikka/service/PaikkaMetadata.java @@ -18,10 +18,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + public record PaikkaMetadata( @JsonProperty("importTimestamp") String importTimestamp, @JsonProperty("dataVersion") String dataVersion, - @JsonProperty("file") String file, + @JsonProperty("file") List files, @JsonProperty("gridLevel") Integer gridLevel, @JsonProperty("paikkaVersion") String paikkaVersion ) { diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java b/src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java index bcaabab..2e06503 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java @@ -91,11 +91,11 @@ private int calculateFileReadWindowSize() { } } - public void importData(String pbfFilePath, String dataDir) throws Exception { + public void importData(List pbfFilePaths, String dataDir) throws Exception { long totalStartTime = System.currentTimeMillis(); - printHeader(pbfFilePath, dataDir); + printHeader(pbfFilePaths, dataDir); - Path pbfFile = Paths.get(pbfFilePath); + List pbfPaths = pbfFilePaths.stream().map(Paths::get).toList(); Path dataDirectory = Paths.get(dataDir); Path tmpDirectory = dataDirectory.resolve("tmp"); dataDirectory.toFile().mkdirs(); @@ -208,16 +208,22 @@ public void importData(String pbfFilePath, String dataDir) throws Exception { stats.printPhaseHeader("PASS 1: Discovery & Indexing"); long pass1Start = System.currentTimeMillis(); stats.setCurrentPhase(1, "1.1.1: Discovery & Indexing"); - pass1DiscoveryAndIndexing(pbfFile, wayIndexDb, neededBoundaryWaysDb, neededNodesDb, relIndexDb, poiIndexDb, stats); + for (Path pbfFile : pbfPaths) { + pass1DiscoveryAndIndexing(pbfFile, wayIndexDb, neededBoundaryWaysDb, neededNodesDb, relIndexDb, poiIndexDb, stats); + } stats.setCurrentPhase(2, "1.1.2: Indexing boundary member ways"); - indexBoundaryMemberWays(pbfFile, neededBoundaryWaysDb, wayIndexDb, neededNodesDb, stats); + for (Path pbfFile : pbfPaths) { + indexBoundaryMemberWays(pbfFile, neededBoundaryWaysDb, wayIndexDb, neededNodesDb, stats); + } stats.printPhaseSummary("PASS 1", pass1Start); // PASS 2: Nodes Cache, Boundaries, POIs stats.printPhaseHeader("PASS 2: Nodes Cache, Boundaries, POIs"); long pass2Start = System.currentTimeMillis(); stats.setCurrentPhase(3, "1.2: Caching node coordinates"); - cacheNeededNodeCoordinates(pbfFile, neededNodesDb, nodeCache, stats); + for (Path pbfFile : pbfPaths) { + cacheNeededNodeCoordinates(pbfFile, neededNodesDb, nodeCache, stats); + } stats.setCurrentPhase(4, "1.3: Processing administrative boundaries"); processAdministrativeBoundariesFromIndex(relIndexDb, nodeCache, wayIndexDb, gridIndexDb, boundariesDb, stats); @@ -260,7 +266,7 @@ public void importData(String pbfFilePath, String dataDir) throws Exception { boundariesDb.flush(new FlushOptions().setWaitForFlush(true)); stats.printFinalStatistics(); stats.printOutcomeAndErrors(); - writeMetadataFile(pbfFile, dataDirectory); + writeMetadataFile(pbfPaths, dataDirectory); } catch (Exception e) { stats.stop(); stats.printError("IMPORT FAILED: " + e.getMessage()); @@ -277,13 +283,13 @@ public void importData(String pbfFilePath, String dataDir) throws Exception { } } - private void writeMetadataFile(Path pbfFile, Path dataDirectory) throws IOException { + private void writeMetadataFile(List pbfFiles, Path dataDirectory) throws IOException { Path metadataPath = dataDirectory.resolve("paikka_metadata.json"); Instant now = Instant.now(); String importTimestamp = DateTimeFormatter.ISO_INSTANT.format(now); String dataVersion = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss").withZone(ZoneOffset.UTC).format(now); ObjectMapper objectMapper = new ObjectMapper(); - PaikkaMetadata metadata = new PaikkaMetadata(importTimestamp, dataVersion, pbfFile.getFileName().toString(), S2Helper.GRID_LEVEL, "1.0.0"); + PaikkaMetadata metadata = new PaikkaMetadata(importTimestamp, dataVersion, pbfFiles.stream().map(path -> path.getFileName().toString()).toList(), S2Helper.GRID_LEVEL, "1.0.0"); objectMapper.writeValue(metadataPath.toFile(), metadata); System.out.println("\n\033[1;32mMetadata file written to: " + metadataPath + "\033[0m"); @@ -1894,9 +1900,10 @@ private String centerText(String text) { return " ".repeat(Math.max(0, pad)) + text; } - private void printHeader(String pbfFilePath, String dataDir) { + private void printHeader(List pbfFilePaths, String dataDir) { System.out.println("\n\033[1;34m" + "=".repeat(80) + "\n" + centerText("PAIKKA IMPORT STARTING") + "\n" + "=".repeat(80) + "\033[0m\n"); - System.out.println("PBF File: " + pbfFilePath); + System.out.println("PBF Files (" + pbfFilePaths.size() + "):"); + pbfFilePaths.forEach(p -> System.out.println(" - " + p)); System.out.println("Data Dir: " + dataDir); System.out.println("Max Import Threads: " + config.getImportConfiguration().getThreads()); long maxHeapBytes = Runtime.getRuntime().maxMemory(); From f17d6faa12858c1dee8428f4652fafdaa65c6387 Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Wed, 29 Apr 2026 10:23:00 +0200 Subject: [PATCH 2/2] feat(#37): add CLI help and import usage instructions, update test and import logic for multi-file support --- .../paikka/PaikkaApplication.java | 70 ++++++++++++++++-- .../paikka/service/ImportServiceTest.java | 9 ++- src/test/resources/data-monaco.zip | Bin 343591 -> 343313 bytes 3 files changed, 71 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java b/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java index d6f27ef..382c99d 100644 --- a/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java +++ b/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java @@ -34,7 +34,14 @@ public class PaikkaApplication implements CommandLineRunner { @Autowired private ImportService importService; - public static void main(String[] args) { + static void main(String[] args) { + for (String arg : args) { + if ("-h".equals(arg) || "--help".equals(arg)) { + printHelp(); + System.exit(0); + } + } + SpringApplication app = new SpringApplication(PaikkaApplication.class); // Check if this is import mode @@ -45,7 +52,7 @@ public static void main(String[] args) { break; } } - + if (isImportMode) { logger.info("Starting in import mode"); app.setWebApplicationType(org.springframework.boot.WebApplicationType.NONE); @@ -57,8 +64,8 @@ public static void main(String[] args) { app.run(args); } } - - private void printApiInfo() { + + private static void printApiInfo() { logger.info("PAIKKA is now serving data under the following endpoints:"); logger.info(" Health Check: GET /api/v1/health"); logger.info(" Reverse Geocoding: GET /api/v1/reverse?lat=60.1699&lon=24.9384"); @@ -110,7 +117,8 @@ public void run(String... args) throws Exception { if (isImportMode) { if (pbfFiles.isEmpty()) { - logger.error("Import mode requires at least one PBF file (use --pbf-file or trailing positional args)"); + logger.error("Import mode requires at least one PBF file"); + printImportUsage(); System.exit(1); } try { @@ -124,4 +132,56 @@ public void run(String... args) throws Exception { printApiInfo(); } } + + private static void printHelp() { + System.out.println("\n=== PAIKKA Help ==="); + System.out.println("\nUsage: java -jar paikka.jar "); + + System.out.println("\nGlobal Options:"); + System.out.println(" -h, --help Show this help message and exit"); + + System.out.println("\nApplication Modes:"); + System.out.println(" 1. API Server Mode (default, no --import flag):"); + System.out.println(" Starts the REST API server for reverse geocoding and geometry queries."); + System.out.println(" Available endpoints after startup:"); + System.out.println(" • Health Check: GET /api/v1/health"); + System.out.println(" • Reverse Geocoding: GET /api/v1/reverse?lat=&lon=[&lang=][&limit=]"); + System.out.println(" • Geometry: GET /api/v1/geometry/"); + System.out.println(" Sample API requests:"); + System.out.println(" curl 'http://localhost:8080/api/v1/health'"); + System.out.println(" curl 'http://localhost:8080/api/v1/reverse?lat=60.1699&lon=24.9384'"); + + System.out.println("\n 2. Import Mode (requires --import flag):"); + System.out.println(" Imports OpenStreetMap PBF files into the Paikka datastore."); + System.out.println(" All specified PBF files are combined into a single final datastore."); + + System.out.println("\nImport Mode Options:"); + System.out.println(" --import Enable import mode (required for data import)"); + System.out.println(" --pbf-file Specify PBF file(s). Supports multiple formats:"); + System.out.println(" • Comma-separated list: --pbf-file \"file1.pbf,file2.pbf\""); + System.out.println(" • Repeated flags: --pbf-file file1.pbf --pbf-file file2.pbf"); + System.out.println(" --data-dir Path to data directory (default: ./data)"); + System.out.println(" Positional arguments (after all flags) are treated as PBF files in import mode"); + + System.out.println("\nImport Examples:"); + System.out.println(" # Single PBF file"); + System.out.println(" java -jar paikka.jar --import --pbf-file /data/osm.pbf"); + System.out.println(" # Multiple PBFs (comma-separated)"); + System.out.println(" java -jar paikka.jar --import --pbf-file \"/data/osm1.pbf,/data/osm2.pbf\" --data-dir ./data"); + System.out.println(" # Multiple PBFs (repeated --pbf-file flags)"); + System.out.println(" java -jar paikka.jar --import --pbf-file /data/osm1.pbf --pbf-file /data/osm2.pbf"); + System.out.println(" # Multiple PBFs (trailing positional arguments)"); + System.out.println(" java -jar paikka.jar --import /data/osm1.pbf /data/osm2.pbf"); + } + + private static void printImportUsage() { + System.out.println("\n=== PAIKKA Import Mode Usage ==="); + System.out.println("\nNo PBF files specified. Use one of the following methods to specify PBF files:"); + System.out.println(" 1. --pbf-file flag (comma-separated values or repeated flags):"); + System.out.println(" --pbf-file /data/osm1.pbf,/data/osm2.pbf"); + System.out.println(" --pbf-file /data/osm1.pbf --pbf-file /data/osm2.pbf"); + System.out.println(" 2. Trailing positional arguments after all flags:"); + System.out.println(" java -jar paikka.jar --import /data/osm1.pbf /data/osm2.pbf"); + System.out.println("\nFor full help, run with -h or --help"); + } } diff --git a/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java b/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java index 13ddb7b..c3df746 100644 --- a/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java +++ b/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java @@ -16,9 +16,11 @@ package com.dedicatedcode.paikka.service; -import com.dedicatedcode.paikka.IntegrationTest; import com.dedicatedcode.paikka.config.PaikkaConfiguration; -import com.dedicatedcode.paikka.flatbuffers.*; +import com.dedicatedcode.paikka.flatbuffers.Address; +import com.dedicatedcode.paikka.flatbuffers.Name; +import com.dedicatedcode.paikka.flatbuffers.POI; +import com.dedicatedcode.paikka.flatbuffers.POIList; import com.dedicatedcode.paikka.service.importer.GeometrySimplificationService; import com.dedicatedcode.paikka.service.importer.ImportService; import org.junit.jupiter.api.AfterEach; @@ -34,6 +36,7 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -66,7 +69,7 @@ void setUp() throws Exception { S2Helper s2Helper = new S2Helper(); ImportService importService = new ImportService(s2Helper, geometrySimplificationService, config); - importService.importData(tempImportFile.toString(), tempDataDir.toString()); + importService.importData(Collections.singletonList(tempImportFile.toString()), tempDataDir.toString()); } @AfterEach diff --git a/src/test/resources/data-monaco.zip b/src/test/resources/data-monaco.zip index 38cbf9f37accb0906bf2177adb3453a250d8666a..2c604be0fde77099ac0c64233a478a0d4682c5a3 100644 GIT binary patch delta 888 zcmZ4fM`YqJkqQ2c4AULWS;aS|FA`y8W)Wdvm_8TCoop&+Op^t~T%cO_Spc;*h<$-_KiB}d3&fqF+yGlv@y+}a^;}T?TT38+m(q6i zdYDa@JmSYz@=Y@uFGBDp+|1lGU zODnh;7+GF0GcbS&-yTCgCPe{;>iEQMY?~YuPD$(Ni%D@$v2}iD!66|1`2Nxj8|KbD zBb}tR^P8qx*4^byeEuR@rvr;Sg{9Qj8hd4A_S#9<#5Fj6xI2I4wi(;w`=XTGjg?IQ z$b@G-_!jO_`MLiL`->3Gj-vSwk3EP?wz$|GRuE*UVW)Tcejt|!$NanE6XYgWdmHjG z0K-lS7}`U4AAS0E1< z<%-Nk;tZ4LiyKd`w_)V~iUH$Yk$ECej8DP}CI*aqMMjk|3`!olBnXVZ&+<`G`J791AWb8P%!s$NdLZh8(=52dCYy@=2dbLCbkWfr zKyx%0v76;8?x)1gv1@zV7k;2B0qiz1Y+fqf$puuGelu;03l{^!a%p7K;C3=>c2GK~ z?!(Tp?OyAd26YC8at-Y27#L;%Z4zNnU?@n;%+5}X&rK~!Oi3(B)XOT)&kGIVWngz% zxju9G{n-;6Me2Re81gYG3ba1l?X!1+B!`P-^V4I8_)@R=ltprQ=k%0MKXmBR)8C)( zDeRnfE&XK8ml@hD1{r&tR&%76G}aWjMSnUsPi9`efn%3#+)mk`Z}EN80v1Y7l-zgX z7<19h$BS2Nv@-YGb#Hp(d9{~^^zSK5n7P0$^7?-BODhEJ;sd#wbOvP7JLfxi&h--uKoolHzB$VvvKf|7YEnZ54K-3@fT7LH`#nMzbLSee?%rA#;(Ff_Xsw7V8C0x=U1GjDe-VCg%` z$Ij9AJ~PJeC<{aQaTbQ@7fo3c8962fh)?%2V|8KVm>y`ws>moleWMwx6(h&wSW$6q zpo_s^Nn_b`K66$(h$7+X3FfSPjDM%+o3lCqg*KS8Dl#XDF-*1>v!2dx!O8*DZJR1S zUEhM0i@8ahVY0utF;slIl?_nsN1)g`aVwbEeG8zybEY#|vf51lV9Cnj2Fu~d@eWF? zFtDYu-iv{lqzkfvYx-PERt2V5@9Fz3SxtdXllK;%&SS;O#kg#`xD~4<%;U?Z7h8cm zK7Ac97?@a7reCyTwMF(Yb59Dx