Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ The [OpenStreetMap website](https://www.openstreetmap.org/export/) allows you to

| Dataset | original | filtered... | time taken | reduction | during import | imported | time taken | reduction |
|------------|----------|-------------|------------|-----------|---------------|----------|------------|-----------|
| Planet | 86 GB | 34 GB | 40 min | ~60% | ~250 GB | ~65 GB | ~11 h | ~25% |
| Planet | 86 GB | 36 GB | 40 min | ~57% | ~250 GB | ~65 GB | ~11 h | ~25% |
| Germany | 4.4 GB | 1.8 GB | 2 min | ~59% | ~14.4 GB | 3,81 GB | ~18 min | ~13% |
| Netherland | 1.4 GB | 394 MB | 30 s | ~70% | ~2,69 GB | 705,7 MB | ~3 min | ~50% |

Expand Down
22 changes: 0 additions & 22 deletions scripts/import.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ usage() {
echo " --data-dir PATH Directory to store processed data (default: ./)"
echo " --memory SIZE JVM heap size (default: 16g)"
echo " --threads NUM Maximum number of import threads (default: half of CPU cores)"
echo " -y, --skip-mem-check Skip memory availability check and warning prompt"
echo " -h, --help Show this help message"
echo ""
echo "Examples:"
Expand All @@ -41,7 +40,6 @@ THREADS=""
PBF_FILE=""

# Parse arguments
SKIP_MEM_CHECK=false
while [[ $# -gt 0 ]]; do
case $1 in
--jar-file)
Expand All @@ -60,10 +58,6 @@ while [[ $# -gt 0 ]]; do
THREADS="$2"
shift 2
;;
-y|--skip-mem-check)
SKIP_MEM_CHECK=true
shift
;;
-h|--help)
usage
;;
Expand Down Expand Up @@ -135,22 +129,6 @@ REQUESTED_MEM_GB=$(echo "$MEMORY" | sed 's/[^0-9]//g')
echo "System memory: ${AVAILABLE_MEM_GB}GB available"
echo "Requested heap: $MEMORY"

if [ "$SKIP_MEM_CHECK" = false ] && [ "$REQUESTED_MEM_GB" -gt "$AVAILABLE_MEM_GB" ]; then
echo ""
echo "⚠️ WARNING: Requested heap size ($MEMORY) exceeds available memory (${AVAILABLE_MEM_GB}GB)"
echo " This may cause the process to be killed by the OOM killer."
echo " Consider reducing heap size or adding more RAM."
echo ""
read -p "Continue anyway? (y/N): " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
echo "Import cancelled."
exit 1
fi
elif [ "$SKIP_MEM_CHECK" = true ]; then
echo "Skipping memory check (--skip-mem-check flag used)"
fi

# Build JVM arguments with memory management optimizations
JVM_ARGS="-Xmx$MEMORY -Xms$MEMORY"
JVM_ARGS="$JVM_ARGS -XX:+UseG1GC"
Expand Down
3 changes: 1 addition & 2 deletions scripts/update.sh
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ local_pull_docker_image() {
# LOCAL: Filters the full planet PBF file using the Paikka container.
###
local_filter_pbf() {
log "LOCAL: Filtering PBF file (approx. 50 mins)"
log "LOCAL: Filtering PBF file"
sudo docker run --rm -v "$DOWNLOAD_DIR":/data "$DOCKER_IMAGE" prepare "/data/$PBF_INPUT_FILE" "/data/$PBF_FILTERED_FILE"
}

Expand All @@ -116,7 +116,6 @@ local_create_import_bundle() {
--memory "$IMPORT_MEMORY" \
--threads "$IMPORT_THREADS" \
--data-dir "/import/" \
--skip-mem-check \
"/download/$PBF_FILTERED_FILE"
}

Expand Down
Loading