Skip to content

37 importing to contriesregions #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dgraf-gh merged 2 commits into from
Apr 29, 2026
Merged

37 importing to contriesregions #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f23a0da
feat(#37): support importing multiple PBF files, update metadata stru…
dgraf-gh Apr 29, 2026
f17d6fa
feat(#37): add CLI help and import usage instructions, update test an…
dgraf-gh Apr 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 97 additions & 19 deletions src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

import java.util.*;

@SpringBootApplication
public class PaikkaApplication implements CommandLineRunner {

Expand All @@ -32,7 +34,14 @@ public class PaikkaApplication implements CommandLineRunner {
@Autowired
private ImportService importService;

public static void main(String[] args) {
static void main(String[] args) {
for (String arg : args) {
if ("-h".equals(arg) || "--help".equals(arg)) {
printHelp();
System.exit(0);
}
}

SpringApplication app = new SpringApplication(PaikkaApplication.class);

// Check if this is import mode
Expand All @@ -43,7 +52,7 @@ public static void main(String[] args) {
break;
}
}

if (isImportMode) {
logger.info("Starting in import mode");
app.setWebApplicationType(org.springframework.boot.WebApplicationType.NONE);
Expand All @@ -55,8 +64,8 @@ public static void main(String[] args) {
app.run(args);
}
}
private void printApiInfo() {

private static void printApiInfo() {
logger.info("PAIKKA is now serving data under the following endpoints:");
logger.info(" Health Check: GET /api/v1/health");
logger.info(" Reverse Geocoding: GET /api/v1/reverse?lat=60.1699&lon=24.9384");
Expand All @@ -70,33 +79,50 @@ private void printApiInfo() {
logger.info(" curl 'http://localhost:8080/api/v1/geometry/12345'");
logger.info("");
}

@Override
public void run(String... args) throws Exception {
// Only run import logic if --import flag is present
boolean isImportMode = false;
String pbfFile = null;
List<String> pbfFiles = new ArrayList<>();
String dataDir = "./data";

Set<Integer> usedArgIndices = new HashSet<>();

// Process flags and their values first
for (int i = 0; i < args.length; i++) {
if ("--import".equals(args[i])) {
String arg = args[i];
if ("--import".equals(arg)) {
isImportMode = true;
} else if ("--pbf-file".equals(args[i]) && i + 1 < args.length) {
pbfFile = args[i + 1];
} else if ("--data-dir".equals(args[i]) && i + 1 < args.length) {
dataDir = args[i + 1];
} else if ("--pbf-file".equals(arg)) {
if (i + 1 >= args.length) { logger.error("Missing --pbf-file value"); System.exit(1); }
String value = args[ i + 1];
usedArgIndices.add(i + 1);
// Split comma-separated values, add non-empty trimmed paths
Arrays.stream(value.split(",")).map(String::trim).filter(s -> !s.isEmpty()).forEach(pbfFiles::add);
i++; // Skip flag value
} else if ("--data-dir".equals(arg)) {
if (i + 1 >= args.length) { logger.error("Missing --data-dir value"); System.exit(1); }
dataDir = args[ i + 1];
usedArgIndices.add(i + 1);
i++; // Skip flag value
}
}


// Collect trailing positional args (not flags/flag values) as PBF files
for (int i = 0; i < args.length; i++) {
if (usedArgIndices.contains(i)) continue;
String arg = args[i];
if (arg.startsWith("--")) continue; // Skip unrecognized flags
if (isImportMode) pbfFiles.add(arg.trim());
}

if (isImportMode) {
if (pbfFile == null) {
logger.error("Import mode requires --pbf-file argument");
if (pbfFiles.isEmpty()) {
logger.error("Import mode requires at least one PBF file");
printImportUsage();
System.exit(1);
}


try {
importService.importData(pbfFile, dataDir);
importService.importData(pbfFiles, dataDir);
System.exit(0);
} catch (Exception e) {
logger.error("Import failed", e);
Expand All @@ -106,4 +132,56 @@ public void run(String... args) throws Exception {
printApiInfo();
}
}

private static void printHelp() {
System.out.println("\n=== PAIKKA Help ===");
System.out.println("\nUsage: java -jar paikka.jar ");

System.out.println("\nGlobal Options:");
System.out.println(" -h, --help Show this help message and exit");

System.out.println("\nApplication Modes:");
System.out.println(" 1. API Server Mode (default, no --import flag):");
System.out.println(" Starts the REST API server for reverse geocoding and geometry queries.");
System.out.println(" Available endpoints after startup:");
System.out.println(" • Health Check: GET /api/v1/health");
System.out.println(" • Reverse Geocoding: GET /api/v1/reverse?lat=<lat>&lon=<lon>[&lang=<lang>][&limit=<limit>]");
System.out.println(" • Geometry: GET /api/v1/geometry/<poiId>");
System.out.println(" Sample API requests:");
System.out.println(" curl 'http://localhost:8080/api/v1/health'");
System.out.println(" curl 'http://localhost:8080/api/v1/reverse?lat=60.1699&lon=24.9384'");

System.out.println("\n 2. Import Mode (requires --import flag):");
System.out.println(" Imports OpenStreetMap PBF files into the Paikka datastore.");
System.out.println(" All specified PBF files are combined into a single final datastore.");

System.out.println("\nImport Mode Options:");
System.out.println(" --import Enable import mode (required for data import)");
System.out.println(" --pbf-file <path> Specify PBF file(s). Supports multiple formats:");
System.out.println(" • Comma-separated list: --pbf-file \"file1.pbf,file2.pbf\"");
System.out.println(" • Repeated flags: --pbf-file file1.pbf --pbf-file file2.pbf");
System.out.println(" --data-dir <path> Path to data directory (default: ./data)");
System.out.println(" Positional arguments (after all flags) are treated as PBF files in import mode");

System.out.println("\nImport Examples:");
System.out.println(" # Single PBF file");
System.out.println(" java -jar paikka.jar --import --pbf-file /data/osm.pbf");
System.out.println(" # Multiple PBFs (comma-separated)");
System.out.println(" java -jar paikka.jar --import --pbf-file \"/data/osm1.pbf,/data/osm2.pbf\" --data-dir ./data");
System.out.println(" # Multiple PBFs (repeated --pbf-file flags)");
System.out.println(" java -jar paikka.jar --import --pbf-file /data/osm1.pbf --pbf-file /data/osm2.pbf");
System.out.println(" # Multiple PBFs (trailing positional arguments)");
System.out.println(" java -jar paikka.jar --import /data/osm1.pbf /data/osm2.pbf");
}

private static void printImportUsage() {
System.out.println("\n=== PAIKKA Import Mode Usage ===");
System.out.println("\nNo PBF files specified. Use one of the following methods to specify PBF files:");
System.out.println(" 1. --pbf-file flag (comma-separated values or repeated flags):");
System.out.println(" --pbf-file /data/osm1.pbf,/data/osm2.pbf");
System.out.println(" --pbf-file /data/osm1.pbf --pbf-file /data/osm2.pbf");
System.out.println(" 2. Trailing positional arguments after all flags:");
System.out.println(" java -jar paikka.jar --import /data/osm1.pbf /data/osm2.pbf");
System.out.println("\nFor full help, run with -h or --help");
}
}
2 changes: 1 addition & 1 deletion src/main/java/com/dedicatedcode/paikka/service/MetadataService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public Map<String, Object> getMetadata() {
Map<String, Object> metadataMap = new HashMap<>();
metadataMap.put("importTimestamp", metadata.importTimestamp());
metadataMap.put("dataVersion", metadata.dataVersion());
metadataMap.put("file", metadata.file());
metadataMap.put("files", metadata.files());
metadataMap.put("gridLevel", metadata.gridLevel());
metadataMap.put("paikkaVersion", metadata.paikkaVersion());
return Collections.unmodifiableMap(metadataMap);
Expand Down
4 changes: 3 additions & 1 deletion src/main/java/com/dedicatedcode/paikka/service/PaikkaMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@

import com.fasterxml.jackson.annotation.JsonProperty;

import java.util.List;

public record PaikkaMetadata(
@JsonProperty("importTimestamp") String importTimestamp,
@JsonProperty("dataVersion") String dataVersion,
@JsonProperty("file") String file,
@JsonProperty("file") List<String> files,
@JsonProperty("gridLevel") Integer gridLevel,
@JsonProperty("paikkaVersion") String paikkaVersion
) {
Expand Down
29 changes: 18 additions & 11 deletions src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,11 @@ private int calculateFileReadWindowSize() {
}
}

public void importData(String pbfFilePath, String dataDir) throws Exception {
public void importData(List<String> pbfFilePaths, String dataDir) throws Exception {
long totalStartTime = System.currentTimeMillis();
printHeader(pbfFilePath, dataDir);
printHeader(pbfFilePaths, dataDir);

Path pbfFile = Paths.get(pbfFilePath);
List<Path> pbfPaths = pbfFilePaths.stream().map(Paths::get).toList();
Path dataDirectory = Paths.get(dataDir);
Path tmpDirectory = dataDirectory.resolve("tmp");
dataDirectory.toFile().mkdirs();
Expand Down Expand Up @@ -208,16 +208,22 @@ public void importData(String pbfFilePath, String dataDir) throws Exception {
stats.printPhaseHeader("PASS 1: Discovery & Indexing");
long pass1Start = System.currentTimeMillis();
stats.setCurrentPhase(1, "1.1.1: Discovery & Indexing");
pass1DiscoveryAndIndexing(pbfFile, wayIndexDb, neededBoundaryWaysDb, neededNodesDb, relIndexDb, poiIndexDb, stats);
for (Path pbfFile : pbfPaths) {
pass1DiscoveryAndIndexing(pbfFile, wayIndexDb, neededBoundaryWaysDb, neededNodesDb, relIndexDb, poiIndexDb, stats);
}
stats.setCurrentPhase(2, "1.1.2: Indexing boundary member ways");
indexBoundaryMemberWays(pbfFile, neededBoundaryWaysDb, wayIndexDb, neededNodesDb, stats);
for (Path pbfFile : pbfPaths) {
indexBoundaryMemberWays(pbfFile, neededBoundaryWaysDb, wayIndexDb, neededNodesDb, stats);
}
stats.printPhaseSummary("PASS 1", pass1Start);

// PASS 2: Nodes Cache, Boundaries, POIs
stats.printPhaseHeader("PASS 2: Nodes Cache, Boundaries, POIs");
long pass2Start = System.currentTimeMillis();
stats.setCurrentPhase(3, "1.2: Caching node coordinates");
cacheNeededNodeCoordinates(pbfFile, neededNodesDb, nodeCache, stats);
for (Path pbfFile : pbfPaths) {
cacheNeededNodeCoordinates(pbfFile, neededNodesDb, nodeCache, stats);
}

stats.setCurrentPhase(4, "1.3: Processing administrative boundaries");
processAdministrativeBoundariesFromIndex(relIndexDb, nodeCache, wayIndexDb, gridIndexDb, boundariesDb, stats);
Expand Down Expand Up @@ -260,7 +266,7 @@ public void importData(String pbfFilePath, String dataDir) throws Exception {
boundariesDb.flush(new FlushOptions().setWaitForFlush(true));
stats.printFinalStatistics();
stats.printOutcomeAndErrors();
writeMetadataFile(pbfFile, dataDirectory);
writeMetadataFile(pbfPaths, dataDirectory);
} catch (Exception e) {
stats.stop();
stats.printError("IMPORT FAILED: " + e.getMessage());
Expand All @@ -277,13 +283,13 @@ public void importData(String pbfFilePath, String dataDir) throws Exception {
}
}

private void writeMetadataFile(Path pbfFile, Path dataDirectory) throws IOException {
private void writeMetadataFile(List<Path> pbfFiles, Path dataDirectory) throws IOException {
Path metadataPath = dataDirectory.resolve("paikka_metadata.json");
Instant now = Instant.now();
String importTimestamp = DateTimeFormatter.ISO_INSTANT.format(now);
String dataVersion = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss").withZone(ZoneOffset.UTC).format(now);
ObjectMapper objectMapper = new ObjectMapper();
PaikkaMetadata metadata = new PaikkaMetadata(importTimestamp, dataVersion, pbfFile.getFileName().toString(), S2Helper.GRID_LEVEL, "1.0.0");
PaikkaMetadata metadata = new PaikkaMetadata(importTimestamp, dataVersion, pbfFiles.stream().map(path -> path.getFileName().toString()).toList(), S2Helper.GRID_LEVEL, "1.0.0");

objectMapper.writeValue(metadataPath.toFile(), metadata);
System.out.println("\n\033[1;32mMetadata file written to: " + metadataPath + "\033[0m");
Expand Down Expand Up @@ -1894,9 +1900,10 @@ private String centerText(String text) {
return " ".repeat(Math.max(0, pad)) + text;
}

private void printHeader(String pbfFilePath, String dataDir) {
private void printHeader(List<String> pbfFilePaths, String dataDir) {
System.out.println("\n\033[1;34m" + "=".repeat(80) + "\n" + centerText("PAIKKA IMPORT STARTING") + "\n" + "=".repeat(80) + "\033[0m\n");
System.out.println("PBF File: " + pbfFilePath);
System.out.println("PBF Files (" + pbfFilePaths.size() + "):");
pbfFilePaths.forEach(p -> System.out.println(" - " + p));
System.out.println("Data Dir: " + dataDir);
System.out.println("Max Import Threads: " + config.getImportConfiguration().getThreads());
long maxHeapBytes = Runtime.getRuntime().maxMemory();
Expand Down
9 changes: 6 additions & 3 deletions src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,11 @@

package com.dedicatedcode.paikka.service;

import com.dedicatedcode.paikka.IntegrationTest;
import com.dedicatedcode.paikka.config.PaikkaConfiguration;
import com.dedicatedcode.paikka.flatbuffers.*;
import com.dedicatedcode.paikka.flatbuffers.Address;
import com.dedicatedcode.paikka.flatbuffers.Name;
import com.dedicatedcode.paikka.flatbuffers.POI;
import com.dedicatedcode.paikka.flatbuffers.POIList;
import com.dedicatedcode.paikka.service.importer.GeometrySimplificationService;
import com.dedicatedcode.paikka.service.importer.ImportService;
import org.junit.jupiter.api.AfterEach;
Expand All @@ -34,6 +36,7 @@
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import static org.junit.jupiter.api.Assertions.*;
Expand Down Expand Up @@ -66,7 +69,7 @@ void setUp() throws Exception {

S2Helper s2Helper = new S2Helper();
ImportService importService = new ImportService(s2Helper, geometrySimplificationService, config);
importService.importData(tempImportFile.toString(), tempDataDir.toString());
importService.importData(Collections.singletonList(tempImportFile.toString()), tempDataDir.toString());
}

@AfterEach
Expand Down
Binary file modified src/test/resources/data-monaco.zip
View file
Open in desktop
Binary file not shown.
Loading