diff --git a/build.sh b/build.sh index d91f4c5..4b37b21 100755 --- a/build.sh +++ b/build.sh @@ -1,73 +1,228 @@ #!/usr/bin/env bash -if [ "$#" -ne 3 ]; then - echo "Usage: $0 " +# Set defaults +MEMORY="${MEMORY:-$(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / 1024 / 1024 / 2))M}" + +# Fetch args +while [[ $# -gt 0 ]]; do + case "$1" in + -m|--memory) + MEMORY="$2" + shift 2 + ;; + -m=*|--memory=*) + MEMORY="${1#*=}" + shift + ;; + -d|--max-depth) + MAX_DEPTH="$2" + shift 2 + ;; + -d=*|--max-depth=*) + MAX_DEPTH="${1#*=}" + shift + ;; + -e|--exclude) + EXCLUDE="$2" + shift 2 + ;; + -e=*|--exclude=*) + EXCLUDE="${1#*=}" + shift + ;; + -t|--test) + TEST=1 + shift + ;; + -*) + echo "Unknown option: $1" + exit 1 + ;; + *) + if [[ -z "$PATH_ARG" ]]; then + PATH_ARG="$1" + elif [[ -z "$MODE" ]]; then + MODE="$1" + else + echo "Unexpected argument: $1" + exit 1 + fi + shift + ;; + esac +done + +# Validate args +if [[ -z "$PATH_ARG" || -z "$MODE" ]]; then + cat < [options] + +Modes: + single Process path as single region + recursive Process all sub-regions under path recursively + +Options: + -m, --memory + RAM allocation given to Planetiler + Default: system RAM / 2 + + -d, --max-depth + Maximum recursion depth (only recursive mode) + + -e, --exclude + Semicolon-separated list of full region paths to exclude from processing + + -t, --test + Dry run mode. Prints all regions that would be processed without generating any output. + +EOF exit 1 fi -#Define functions +case "$MODE" in + single|recursive) + ;; + *) + echo "Error: invalid mode '$MODE'" + echo "Valid modes are: single, subreg, recursive" + exit 1 + ;; +esac + +if [[ -n "$MAX_DEPTH" ]]; then + if ! [[ "$MAX_DEPTH" =~ ^[0-9]+$ ]]; then + echo "Error: --max-depth must be a non-negative integer" + exit 1 + fi +fi + +if [[ -n "$MAX_DEPTH" && "$MODE" != "recursive" ]]; then + echo "Warning: --max-depth can only work with recursive, ignoring..." +fi + +echo "Processing path '$PATH_ARG' in $MODE mode with $MEMORY memory allocated..." + +# Define functions +should_skip() { + local path="$1" + + [[ -z "$EXCLUDE" ]] && return 1 + + IFS=';' read -ra EXCL <<< "$EXCLUDE" + for ex in "${EXCL[@]}"; do + [[ "$path" == *"$ex"* ]] && return 0 + done + + return 1 +} + generate_region() { - local REGION="$1" - local REGION_ROOT="$2" - local MEMORY="${3:-8g}" + local PATH_ARG="$1" + local MEMORY="$2" - local POLY_FILE="${REGION}.poly" + PATH_ARG="${PATH_ARG%/}" - mkdir -p work - wget -O "work/$POLY_FILE" \ - "https://download.geofabrik.de/${REGION_ROOT}/${REGION}.poly" + if should_skip "$PATH_ARG"; then + echo "Skipping excluded region: $PATH_ARG" + return 0 + fi - mkdir -p "work/contours/$REGION_ROOT/$REGION" + echo "Generating region: $PATH_ARG" + if [[ "$TEST" == "1" ]]; then + return 0 + fi + + mkdir -p "./work/poly/${PATH_ARG%/*}" + wget "https://download.geofabrik.de/$PATH_ARG.poly" -O "./work/poly/$PATH_ARG.poly" + + mkdir -p "./work/contours/${PATH_ARG%/*}" pyhgtmap \ - --polygon="work/$POLY_FILE" \ - --step=75 \ + --polygon="./work/poly/$PATH_ARG.poly" \ + --step=100 \ --hgtdir=work/hgt \ --sources=view1,view3 \ --simplifyContoursEpsilon=0.001 \ -j16 \ --max-nodes-per-tile=0 \ - --output-prefix="work/contours/$REGION_ROOT/$REGION/con" - - rm -f work/contours.osm + --output-prefix="./work/contours/$PATH_ARG/con" + mkdir -p "./work/tmp" # max-nodes-per-tile=0 SHOULD generate only one file - mv "work/contours/$REGION_ROOT/$REGION"/con* work/contours.osm + # still very much wonky though + mv "work/contours/$PATH_ARG"/con* work/tmp/contours.osm - rm -f data/contours.geojson - - osmium export work/contours.osm \ - -o data/contours.geojson \ + mkdir -p "./data/contours/${PATH_ARG%/*}" + osmium export work/tmp/contours.osm \ + -o data/contours/${PATH_ARG}.geojson \ --overwrite - rm -f work/contours.osm + rm -f work/tmp/contours.osm - mkdir -p "./out/$REGION_ROOT" + mkdir -p "./out/${PATH_ARG%/*}" + + mkdir -p "./data/osm/${PATH_ARG%/*}" + wget "https://download.geofabrik.de/${PATH_ARG%/*}/$( + curl -s "https://download.geofabrik.de/${PATH_ARG%/*}/" | + grep -oP 'href="\K[^"]+' | + grep -vE '^\?C=|/icons/|Parent Directory|^/?$' | + sed 's|/$||' | + grep '\.osm\.pbf$' | + grep -v '\.md5$' | + grep "${PATH_ARG##*/}" | + sort | + tail -n 1 + )" -O "./data/osm/${PATH_ARG}.osm.pbf" java -Xmx"$MEMORY" \ -jar ./bin/planetiler.jar schema.yml \ --download \ - --area="${REGION}" \ - --output="./out/${REGION_ROOT}/${REGION}.mbtiles" \ + --osm_file="./data/osm/${PATH_ARG}.osm.pbf" \ + --contour_file="./data/contours/${PATH_ARG}.geojson" \ + --output="./out/${PATH_ARG}.mbtiles" \ --no-simplify \ --simplify-tolerance-at-max-zoom=0 \ --no-feature-merge \ --simplify-tolerance=0 } -generate_root() { - local REGION_ROOT="$1" - local REGION="$2" - if [[ "$REGION" == "each" ]]; then - while IFS= read -r REGION; do - generate_region "$REGION" "$REGION_ROOT" "$MEMORY" - done < "defs/regions/roots/${REGION_ROOT}.txt" +fetch_path() { + local PATH_ARG="$1" + curl -s "https://download.geofabrik.de/$PATH_ARG/" | + grep -oP 'href="\K[^"]+' | + grep -vE '^\?C=|/icons/|Parent Directory|^/?$' | + sed 's|/$||' | + grep ".poly" | + sed 's/\.poly$//' +} + +all_path() { + local PATH_ARG="$1" + if [ "$PATH_ARG" = "planet" ]; then + PATH_ARG="" + SUBS=$'africa\nantarctica\nasia\naustralia-oceania\ncentral-america\neurope\nnorth-america\nsouth-america' else - generate_region "$REGION" "$REGION_ROOT" "$MEMORY" + SUBS=$(fetch_path "$PATH_ARG") + fi + + if [ -z "$SUBS" ]; then + while IFS= read -r REG; do + generate_region "$PATH_ARG" "$MEMORY" + done <<< "$SUBS" + else + while IFS= read -r REG; do + if [[ -n "$PATH_ARG" ]]; then + all_path "$PATH_ARG/$REG" + else + all_path "$REG" + fi + done <<< "$SUBS" fi } -#Prepare environment +# Prepare environment if [ ! -f ./bin/planetiler.jar ]; then mkdir -p ./bin wget -O ./bin/planetiler.jar https://github.com/onthegomap/planetiler/releases/latest/download/planetiler.jar @@ -84,19 +239,13 @@ else source "$VENV_DIR/bin/activate" fi -REGION_ROOT="$1" -REGION="$2" -MEMORY="$3" - -#Generate -if [ "$REGION_ROOT" == "each" ]; then - if [ "$REGION" != "each" ]; then - echo "Cannot generate 'each' root region for non-each region, exiting..." - exit 1 +# Generate +if [ "$MODE" == "single" ]; then + if [ "$PATH_ARG" = "planet" ]; then + echo "TODO" fi - while IFS= read -r REGION_ROOT; do - generate_root "$REGION_ROOT" "each" "$MEMORY" - done < "defs/regions/planet.txt" -else - generate_root "$REGION_ROOT" "$REGION" "$MEMORY" + generate_region "$PATH_ARG" "$MEMORY" + exit 0 +elif [[ "$MODE" == "recursive" ]]; then + all_path "$PATH_ARG" fi diff --git a/defs/README.md b/defs/README.md deleted file mode 100644 index 47c3059..0000000 --- a/defs/README.md +++ /dev/null @@ -1 +0,0 @@ -Local definitions diff --git a/defs/regions/.europe.txt.kate-swp b/defs/regions/.europe.txt.kate-swp deleted file mode 100644 index bd506a9..0000000 Binary files a/defs/regions/.europe.txt.kate-swp and /dev/null differ diff --git a/defs/regions/planet.txt b/defs/regions/planet.txt deleted file mode 100644 index 8caa993..0000000 --- a/defs/regions/planet.txt +++ /dev/null @@ -1,8 +0,0 @@ -africa -antarctica -asia -australia-oceania -central-america -europe -north-america -south-america diff --git a/defs/regions/roots/africa.txt b/defs/regions/roots/africa.txt deleted file mode 100644 index bc37157..0000000 --- a/defs/regions/roots/africa.txt +++ /dev/null @@ -1,55 +0,0 @@ -algeria -angola -benin -botswana -burkina-faso -burundi -cameroon -canary-islands -cape-verde -central-african-republic -chad -comores -congo-brazzaville -congo-democratic-republic -djibouti -egypt -equatorial-guinea -eritrea -ethiopia -gabon -ghana -guinea -guinea-bissau -ivory-coast -kenya -lesotho -liberia -libya -madagascar -malawi -mali -mauritania -mauritius -morocco -mozambique -namibia -niger -nigeria -rwanda -saint-helena-ascension-and-tristan-da-cunha -sao-tome-and-principe -senegal-and-gambia -seychelles -sierra-leone -somalia -south-africa -south-sudan -sudan -swaziland -tanzania -togo -tunisia -uganda -zambia -zimbabwe diff --git a/defs/regions/roots/asia.txt b/defs/regions/roots/asia.txt deleted file mode 100644 index 3b9ded8..0000000 --- a/defs/regions/roots/asia.txt +++ /dev/null @@ -1,38 +0,0 @@ -afghanistan -armenia -azerbaijan -bangladesh -bhutan -cambodia -china -east-timor -gcc-states -india -indonesia -iran -iraq -israel-and-palestine -japan -jordan -kazakhstan -kyrgyzstan -laos -lebanon -malaysia-singapore-brunei -maldives -mongolia -myanmar -nepal -north-korea -pakistan -philippines -south-korea -sri-lanka -syria -taiwan -tajikistan -thailand -turkmenistan -uzbekistan -vietnam -yemen diff --git a/defs/regions/roots/australia-oceania.txt b/defs/regions/roots/australia-oceania.txt deleted file mode 100644 index 16d41a0..0000000 --- a/defs/regions/roots/australia-oceania.txt +++ /dev/null @@ -1,23 +0,0 @@ -american-oceania -australia -cook-islands -fiji -ile-de-clipperton -kiribati -marshall-islands -micronesia -nauru -new-caledonia -new-zealand -niue -palau -papua-new-guinea -pitcairn-islands -polynesie-francaise -samoa -solomon-islands -tokelau -tonga -tuvalu -vanuatu -wallis-et-futuna diff --git a/defs/regions/roots/central-america.txt b/defs/regions/roots/central-america.txt deleted file mode 100644 index 5823b80..0000000 --- a/defs/regions/roots/central-america.txt +++ /dev/null @@ -1,11 +0,0 @@ -bahamas -belize -costa-rica -cuba -el-salvador -guatemala -haiti-and-domrep -honduras -jamaica -nicaragua -panama diff --git a/defs/regions/roots/europe.txt b/defs/regions/roots/europe.txt deleted file mode 100644 index 79b5ac4..0000000 --- a/defs/regions/roots/europe.txt +++ /dev/null @@ -1,52 +0,0 @@ -albania -andorra -austria -azores -belarus -belgium -bosnia-herzegovina -britain-and-ireland -bulgaria -croatia -cyprus -czech-republic -dach -denmark -estonia -faroe-islands -finland -france -georgia -germany -great-britain -greece -guernsey-jersey -hungary -iceland -ireland-and-northern-ireland -isle-of-man -italy -kosovo -latvia -liechtenstein -lithuania -luxembourg -macedonia -malta -moldova -monaco -montenegro -netherlands -norway -poland -portugal -romania -serbia -slovakia -slovenia -spain -sweden -switzerland -turkey -ukraine -united-kingdom diff --git a/defs/regions/roots/north-america.txt b/defs/regions/roots/north-america.txt deleted file mode 100644 index 3f5c46e..0000000 --- a/defs/regions/roots/north-america.txt +++ /dev/null @@ -1,9 +0,0 @@ -canada -greenland -mexico -us -us-midwest -us-northeast -us-pacific -us-south -us-west diff --git a/defs/regions/roots/south-america.txt b/defs/regions/roots/south-america.txt deleted file mode 100644 index d315893..0000000 --- a/defs/regions/roots/south-america.txt +++ /dev/null @@ -1,12 +0,0 @@ -argentina -bolivia -brazil -chile -colombia -ecuador -guyana -paraguay -peru -suriname -uruguay -venezuela diff --git a/schema.yml b/schema.yml index c96abfd..5e3c2b5 100755 --- a/schema.yml +++ b/schema.yml @@ -2,26 +2,26 @@ schema_name: StrikeMaps Tileschema schema_description: A tileschema for OpenStreetMap data developed by the Strike Maps project. attribution: © OpenStreetMap contributors, Natural Earth, NASA SRTM args: - area: - description: Geofabrik area to download - default: monaco - osm_url: - description: OSM URL to download - default: '${ args.area == "planet" ? "aws:latest" : ("geofabrik:" + args.area) }' + osm_file: + description: Source .osm.pbf file + default: ./source.osm.pbf + contour_file: + description: Contour data geojson + default: ./contours.geojson sources: + osm: + type: osm + local_path: '${ args.osm_file }' ocean: type: shapefile url: https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip - osm: - type: osm - url: '${ args.osm_url }' natural_earth: type: geopackage url: "https://naciscdn.org/naturalearth/packages/natural_earth_vector.gpkg.zip" projection: EPSG:4326 contour: type: geojson - local_path: ./data/contours.geojson + local_path: '${ args.contour_file }' hillshade_json: type: geojson url: "https://raw.githubusercontent.com/tinius/hillshade-json/refs/heads/master/hillshade.json"