Fancy path system

build.sh

@@ -1,69 +1,224 @@
 #!/usr/bin/env bash
 
-if [ "$#" -ne 3 ]; then
+# Set defaults
-    echo "Usage: $0 <region_root> <region> <memory>"
+MEMORY="${MEMORY:-$(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / 1024 / 1024 / 2))M}"
+
+# Fetch args
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -m|--memory)
+            MEMORY="$2"
+            shift 2
+            ;;
+        -m=*|--memory=*)
+            MEMORY="${1#*=}"
+            shift
+            ;;
+        -d|--max-depth)
+            MAX_DEPTH="$2"
+            shift 2
+            ;;
+        -d=*|--max-depth=*)
+            MAX_DEPTH="${1#*=}"
+            shift
+            ;;
+        -e|--exclude)
+            EXCLUDE="$2"
+            shift 2
+            ;;
+        -e=*|--exclude=*)
+            EXCLUDE="${1#*=}"
+            shift
+            ;;
+        -t|--test)
+            TEST=1
+            shift
+            ;;
+        -*)
+            echo "Unknown option: $1"
+            exit 1
+            ;;
+        *)
+            if [[ -z "$PATH_ARG" ]]; then
+                PATH_ARG="$1"
+            elif [[ -z "$MODE" ]]; then
+                MODE="$1"
+            else
+                echo "Unexpected argument: $1"
+                exit 1
+            fi
+            shift
+            ;;
+    esac
+done
+
+# Validate args
+if [[ -z "$PATH_ARG" || -z "$MODE" ]]; then
+    cat <<EOF
+
+Usage:
+  $(basename "$0") <path> <mode> [options]
+
+Modes:
+  single       Process path as single region
+  recursive    Process all sub-regions under path recursively
+
+Options:
+  -m, --memory <size>
+      RAM allocation given to Planetiler
+      Default: system RAM / 2
+
+  -d, --max-depth <n>
+      Maximum recursion depth (only recursive mode)
+
+  -e, --exclude <regions>
+      Semicolon-separated list of full region paths to exclude from processing
+
+  -t, --test
+      Dry run mode. Prints all regions that would be processed without generating any output.
+
+EOF
     exit 1
 fi
 
+case "$MODE" in
+    single|recursive)
+        ;;
+    *)
+        echo "Error: invalid mode '$MODE'"
+        echo "Valid modes are: single, subreg, recursive"
+        exit 1
+        ;;
+esac
+
+if [[ -n "$MAX_DEPTH" ]]; then
+    if ! [[ "$MAX_DEPTH" =~ ^[0-9]+$ ]]; then
+        echo "Error: --max-depth must be a non-negative integer"
+        exit 1
+    fi
+fi
+
+if [[ -n "$MAX_DEPTH" && "$MODE" != "recursive" ]]; then
+    echo "Warning: --max-depth can only work with recursive, ignoring..."
+fi
+
+echo "Processing path '$PATH_ARG' in $MODE mode with $MEMORY memory allocated..."
+
 # Define functions
+should_skip() {
+  local path="$1"
+
+  [[ -z "$EXCLUDE" ]] && return 1
+
+  IFS=';' read -ra EXCL <<< "$EXCLUDE"
+  for ex in "${EXCL[@]}"; do
+    [[ "$path" == *"$ex"* ]] && return 0
+  done
+
+  return 1
+}
+
 generate_region() {
-  local REGION="$1"
+  local PATH_ARG="$1"
-  local REGION_ROOT="$2"
+  local MEMORY="$2"
-  local MEMORY="${3:-8g}"
 
-  local POLY_FILE="${REGION}.poly"
+  PATH_ARG="${PATH_ARG%/}"
 
-  mkdir -p work
+  if should_skip "$PATH_ARG"; then
-  wget -O "work/$POLY_FILE" \
+    echo "Skipping excluded region: $PATH_ARG"
-    "https://download.geofabrik.de/${REGION_ROOT}/${REGION}.poly"
+    return 0
+  fi
 
-  mkdir -p "work/contours/$REGION_ROOT/$REGION"
+  echo "Generating region: $PATH_ARG"
 
+  if [[ "$TEST" == "1" ]]; then
+     return 0
+  fi
+
+  mkdir -p "./work/poly/${PATH_ARG%/*}"
+  wget "https://download.geofabrik.de/$PATH_ARG.poly" -O "./work/poly/$PATH_ARG.poly"
+
+  mkdir -p "./work/contours/${PATH_ARG%/*}"
   pyhgtmap \
-    --polygon="work/$POLY_FILE" \
+    --polygon="./work/poly/$PATH_ARG.poly" \
-    --step=75 \
+    --step=100 \
     --hgtdir=work/hgt \
     --sources=view1,view3 \
     --simplifyContoursEpsilon=0.001 \
     -j16 \
     --max-nodes-per-tile=0 \
-    --output-prefix="work/contours/$REGION_ROOT/$REGION/con"
+    --output-prefix="./work/contours/$PATH_ARG/con"
-
-  rm -f work/contours.osm
 
+  mkdir -p "./work/tmp"
   # max-nodes-per-tile=0 SHOULD generate only one file
-  mv "work/contours/$REGION_ROOT/$REGION"/con* work/contours.osm
+  # still very much wonky though
+  mv "work/contours/$PATH_ARG"/con* work/tmp/contours.osm
 
-  rm -f data/contours.geojson
+  mkdir -p "./data/contours/${PATH_ARG%/*}"
-
+  osmium export work/tmp/contours.osm \
-  osmium export work/contours.osm \
+    -o data/contours/${PATH_ARG}.geojson \
-    -o data/contours.geojson \
     --overwrite
 
-  rm -f work/contours.osm
+  rm -f work/tmp/contours.osm
 
-  mkdir -p "./out/$REGION_ROOT"
+  mkdir -p "./out/${PATH_ARG%/*}"
+
+  mkdir -p "./data/osm/${PATH_ARG%/*}"
+  wget "https://download.geofabrik.de/${PATH_ARG%/*}/$(
+        curl -s "https://download.geofabrik.de/${PATH_ARG%/*}/" |
+        grep -oP 'href="\K[^"]+' |
+        grep -vE '^\?C=|/icons/|Parent Directory|^/?$' |
+        sed 's|/$||' |
+        grep '\.osm\.pbf$' |
+        grep -v '\.md5$' |
+        grep "${PATH_ARG##*/}" |
+        sort |
+        tail -n 1
+    )" -O "./data/osm/${PATH_ARG}.osm.pbf"
 
   java -Xmx"$MEMORY" \
     -jar ./bin/planetiler.jar schema.yml \
     --download \
-    --area="${REGION}" \
+    --osm_file="./data/osm/${PATH_ARG}.osm.pbf" \
-    --output="./out/${REGION_ROOT}/${REGION}.mbtiles" \
+    --contour_file="./data/contours/${PATH_ARG}.geojson" \
+    --output="./out/${PATH_ARG}.mbtiles" \
     --no-simplify \
     --simplify-tolerance-at-max-zoom=0 \
     --no-feature-merge \
     --simplify-tolerance=0
 }
 
-generate_root() {
+fetch_path() {
-    local REGION_ROOT="$1"
+    local PATH_ARG="$1"
-    local REGION="$2"
+    curl -s "https://download.geofabrik.de/$PATH_ARG/" |
-    if [[ "$REGION" == "each" ]]; then
+    grep -oP 'href="\K[^"]+' |
-        while IFS= read -r REGION; do
+    grep -vE '^\?C=|/icons/|Parent Directory|^/?$' |
-            generate_region "$REGION" "$REGION_ROOT" "$MEMORY"
+    sed 's|/$||' |
-        done < "defs/regions/roots/${REGION_ROOT}.txt"
+    grep ".poly" |
+    sed 's/\.poly$//'
+}
+
+all_path() {
+    local PATH_ARG="$1"
+    if [ "$PATH_ARG" = "planet" ]; then
+        PATH_ARG=""
+        SUBS=$'africa\nantarctica\nasia\naustralia-oceania\ncentral-america\neurope\nnorth-america\nsouth-america'
     else
-        generate_region "$REGION" "$REGION_ROOT" "$MEMORY"
+        SUBS=$(fetch_path "$PATH_ARG")
+    fi
+
+    if [ -z "$SUBS" ]; then
+        while IFS= read -r REG; do
+            generate_region "$PATH_ARG" "$MEMORY"
+        done <<< "$SUBS"
+    else
+        while IFS= read -r REG; do
+            if [[ -n "$PATH_ARG" ]]; then
+                all_path "$PATH_ARG/$REG"
+            else
+                all_path "$REG"
+            fi
+        done <<< "$SUBS"
     fi
 }
 
@@ -84,19 +239,13 @@ else
     source "$VENV_DIR/bin/activate"
 fi
 
-REGION_ROOT="$1"
-REGION="$2"
-MEMORY="$3"
-
 # Generate
-if [ "$REGION_ROOT" == "each" ]; then
+if [ "$MODE" == "single" ]; then
-    if [ "$REGION" != "each" ]; then
+    if [ "$PATH_ARG" = "planet" ]; then
-        echo "Cannot generate 'each' root region for non-each region, exiting..."
+        echo "TODO"
-        exit 1
     fi
-    while IFS= read -r REGION_ROOT; do
+    generate_region "$PATH_ARG" "$MEMORY"
-        generate_root "$REGION_ROOT" "each" "$MEMORY"
+    exit 0
-    done < "defs/regions/planet.txt"
+elif [[ "$MODE" == "recursive" ]]; then
-else
+    all_path "$PATH_ARG"
-    generate_root "$REGION_ROOT" "$REGION" "$MEMORY"
 fi

defs/README.md

@@ -1 +0,0 @@
-Local definitions

defs/regions/planet.txt

@@ -1,8 +0,0 @@
-africa
-antarctica
-asia
-australia-oceania
-central-america
-europe
-north-america
-south-america

defs/regions/roots/africa.txt

@@ -1,55 +0,0 @@
-algeria
-angola
-benin
-botswana
-burkina-faso
-burundi
-cameroon
-canary-islands
-cape-verde
-central-african-republic
-chad
-comores
-congo-brazzaville
-congo-democratic-republic
-djibouti
-egypt
-equatorial-guinea
-eritrea
-ethiopia
-gabon
-ghana
-guinea
-guinea-bissau
-ivory-coast
-kenya
-lesotho
-liberia
-libya
-madagascar
-malawi
-mali
-mauritania
-mauritius
-morocco
-mozambique
-namibia
-niger
-nigeria
-rwanda
-saint-helena-ascension-and-tristan-da-cunha
-sao-tome-and-principe
-senegal-and-gambia
-seychelles
-sierra-leone
-somalia
-south-africa
-south-sudan
-sudan
-swaziland
-tanzania
-togo
-tunisia
-uganda
-zambia
-zimbabwe

defs/regions/roots/asia.txt

@@ -1,38 +0,0 @@
-afghanistan
-armenia
-azerbaijan
-bangladesh
-bhutan
-cambodia
-china
-east-timor
-gcc-states
-india
-indonesia
-iran
-iraq
-israel-and-palestine
-japan
-jordan
-kazakhstan
-kyrgyzstan
-laos
-lebanon
-malaysia-singapore-brunei
-maldives
-mongolia
-myanmar
-nepal
-north-korea
-pakistan
-philippines
-south-korea
-sri-lanka
-syria
-taiwan
-tajikistan
-thailand
-turkmenistan
-uzbekistan
-vietnam
-yemen

defs/regions/roots/australia-oceania.txt

@@ -1,23 +0,0 @@
-american-oceania
-australia
-cook-islands
-fiji
-ile-de-clipperton
-kiribati
-marshall-islands
-micronesia
-nauru
-new-caledonia
-new-zealand
-niue
-palau
-papua-new-guinea
-pitcairn-islands
-polynesie-francaise
-samoa
-solomon-islands
-tokelau
-tonga
-tuvalu
-vanuatu
-wallis-et-futuna

defs/regions/roots/central-america.txt

@@ -1,11 +0,0 @@
-bahamas
-belize
-costa-rica
-cuba
-el-salvador
-guatemala
-haiti-and-domrep
-honduras
-jamaica
-nicaragua
-panama

defs/regions/roots/europe.txt

@@ -1,52 +0,0 @@
-albania
-andorra
-austria
-azores
-belarus
-belgium
-bosnia-herzegovina
-britain-and-ireland
-bulgaria
-croatia
-cyprus
-czech-republic
-dach
-denmark
-estonia
-faroe-islands
-finland
-france
-georgia
-germany
-great-britain
-greece
-guernsey-jersey
-hungary
-iceland
-ireland-and-northern-ireland
-isle-of-man
-italy
-kosovo
-latvia
-liechtenstein
-lithuania
-luxembourg
-macedonia
-malta
-moldova
-monaco
-montenegro
-netherlands
-norway
-poland
-portugal
-romania
-serbia
-slovakia
-slovenia
-spain
-sweden
-switzerland
-turkey
-ukraine
-united-kingdom

defs/regions/roots/north-america.txt

@@ -1,9 +0,0 @@
-canada
-greenland
-mexico
-us
-us-midwest
-us-northeast
-us-pacific
-us-south
-us-west

defs/regions/roots/south-america.txt

@@ -1,12 +0,0 @@
-argentina
-bolivia
-brazil
-chile
-colombia
-ecuador
-guyana
-paraguay
-peru
-suriname
-uruguay
-venezuela

schema.yml

@@ -2,26 +2,26 @@ schema_name: StrikeMaps Tileschema
 schema_description: A tileschema for OpenStreetMap data developed by the Strike Maps project.
 attribution: <a href="https://www.openstreetmap.org/copyright" target="_blank">&copy; OpenStreetMap contributors</a>, Natural Earth, NASA SRTM
 args:
-  area:
+  osm_file:
-    description: Geofabrik area to download
+    description: Source .osm.pbf file
-    default: monaco
+    default: ./source.osm.pbf
-  osm_url:
+  contour_file:
-    description: OSM URL to download
+    description: Contour data geojson
-    default: '${ args.area == "planet" ? "aws:latest" : ("geofabrik:" + args.area) }'
+    default: ./contours.geojson
 sources:
+  osm:
+    type: osm
+    local_path: '${ args.osm_file }'
   ocean:
     type: shapefile
     url: https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip
-  osm:
-    type: osm
-    url: '${ args.osm_url }'
   natural_earth:
     type: geopackage
     url: "https://naciscdn.org/naturalearth/packages/natural_earth_vector.gpkg.zip"
     projection: EPSG:4326
   contour:
     type: geojson
-    local_path: ./data/contours.geojson
+    local_path: '${ args.contour_file }'
   hillshade_json:
     type: geojson
     url: "https://raw.githubusercontent.com/tinius/hillshade-json/refs/heads/master/hillshade.json"