Apply OpenMapTilev3.14 changes

.env

@@ -1,13 +1,54 @@
-POSTGRES_DB=openmaptiles
-POSTGRES_USER=openmaptiles
-POSTGRES_PASSWORD=openmaptiles
-POSTGRES_HOST=postgres
-POSTGRES_PORT=5432
-QUICKSTART_MIN_ZOOM=0
-QUICKSTART_MAX_ZOOM=7
-DIFF_MODE=false
-TOOLS_VERSION=3.1.0
+# This file defines default environment variables for all images
+
+# Layers definition and meta data
+TILESET_FILE=openmaptiles.yaml
+
+# Use 3-part patch version to ignore patch updates, e.g. 7.0.0
+TOOLS_VERSION=7.0
+
+# Make sure these values are in sync with the ones in .env-postgres file
+PGDATABASE=openmaptiles
+PGUSER=openmaptiles
+PGPASSWORD=openmaptiles
+PGHOST=postgres
+PGPORT=5432
 
+# BBOX may get overwritten by the computed bbox of the specific area:
+#   make generate-bbox-file
+# By default, the Makefile will use the content of data/$(area).bbox file if it exists.
 BBOX=-180.0,-85.0511,180.0,85.0511
+
+# Which zooms to generate with   make generate-tiles-pg
 MIN_ZOOM=0
-MAX_ZOOM=14
+MAX_ZOOM=7
+
+# `MID_ZOOM` setting only works with `make generate-tiles-pg` command.  Make sure MID_ZOOM < MAX_ZOOM.
+# See https://github.com/openmaptiles/openmaptiles-tools/pull/383
+# MID_ZOOM=11
+
+# Use  true  (case sensitive) to allow data updates
+DIFF_MODE=false
+
+# Some area data like openstreetmap.fr can contain invalid references
+# that must be cleaned up before using it for borders -- set it to true.
+BORDERS_CLEANUP=false
+
+# The current setup assumes this file is placed inside the data/ dir
+MBTILES_FILE=tiles.mbtiles
+# This is the current repl_config.json location, pre-configured in the tools Dockerfile
+# Makefile and quickstart replace it with the dynamically generated one, but we keep it here in case some other method is used to run.
+IMPOSM_CONFIG_FILE=/usr/src/app/config/repl_config.json
+
+# import-borders temp files - set them here to defaults, and override in the Makefile based on the area
+BORDERS_CLEANUP_FILE=data/borders/cleanup.pbf
+BORDERS_PBF_FILE=data/borders/filtered.pbf
+BORDERS_CSV_FILE=data/borders/lines.csv
+
+# Number of parallel processes to use when importing sql files
+MAX_PARALLEL_PSQL=5
+
+# Number of parallel threads to use when generating vector map tiles
+COPY_CONCURRENCY=10
+
+# Variables for generate tiles using tilelive-pgquery
+PGHOSTS_LIST=

.github/workflows/build-test-data.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+#
+# Download several areas, combine them into a single file, and print --bbox params needed to run test-perf
+#
+
+# List of Geofabrik areas
+TEST_AREAS=(equatorial-guinea liechtenstein district-of-columbia greater-london)
+
+: "${DATA_DIR:=/tileset/data/test}"
+: "${DATA_FILE_SUFFIX:=-latest.osm.pbf}"
+: "${RESULT_FILE:=test${DATA_FILE_SUFFIX}}"
+
+mkdir -p "$DATA_DIR"
+cd "$DATA_DIR"
+
+
+echo -e $"\n=========== downloading areas" "${TEST_AREAS[@]}" "==========================="
+for area in "${TEST_AREAS[@]}"; do
+  file="${area}${DATA_FILE_SUFFIX}"
+  if [ -f "$file" ]; then
+    echo "File $file already exists, skipping download"
+  else
+    download-osm geofabrik "${area}" -- -d "$DATA_DIR"
+    if [ ! -f "$file" ]; then
+      echo "Unexpected error while downloading $file, aborting"
+      exit 1
+    fi
+  fi
+done
+
+
+echo -e $"\n=========== Merging" "${TEST_AREAS[@]}" "into ${RESULT_FILE} ====="
+rm -f "${RESULT_FILE}"
+OSMOSIS_ARG="--read-pbf ${TEST_AREAS[0]}${DATA_FILE_SUFFIX} $(printf " --read-pbf %s${DATA_FILE_SUFFIX} --merge" "${TEST_AREAS[@]:1}")"
+# shellcheck disable=SC2086
+( set -x; osmosis ${OSMOSIS_ARG} --write-pbf "${RESULT_FILE}" )
+
+
+echo -e $"\n=========== Computing test BBOXes ======================="
+echo -e $"\n  File ${RESULT_FILE} ($(du -b "$RESULT_FILE" | cut -f1)) has been generated with these test areas:\n"
+for area in "${TEST_AREAS[@]}"; do
+  file="${area}${DATA_FILE_SUFFIX}"
+  STATS=$(osmconvert --out-statistics "$file" )
+  LON_MIN=$( echo "$STATS" | grep "lon min:" | cut -d":" -f 2 | awk '{gsub(/^ +| +$/,"")} {print $0}' )
+  LON_MAX=$( echo "$STATS" | grep "lon max:" | cut -d":" -f 2 | awk '{gsub(/^ +| +$/,"")} {print $0}' )
+  LAT_MIN=$( echo "$STATS" | grep "lat min:" | cut -d":" -f 2 | awk '{gsub(/^ +| +$/,"")} {print $0}' )
+  LAT_MAX=$( echo "$STATS" | grep "lat max:" | cut -d":" -f 2 | awk '{gsub(/^ +| +$/,"")} {print $0}' )
+  BBOX="${LON_MIN},${LAT_MIN},${LON_MAX},${LAT_MAX}"
+  FILE_SIZE="$(du -b "$file" | cut -f1)"
+
+  cat <<EOF | (PYTHONPATH=/usr/src/app python)
+from openmaptiles.perfutils import TestCase
+tc = TestCase('${area}', 'a', bbox='${BBOX}')
+info = f"# {tc.id} {tc.size():,} tiles at z14, \
+{$FILE_SIZE/1024/1024:,.1f}MB, {$FILE_SIZE/tc.size():,.1f} bytes/tile \
+[{tc.start[0]}/{tc.start[1]}]x[{tc.before[0] - 1}/{tc.before[1] - 1}]"
+print(f"  --bbox {tc.bbox:46} {info}")
+EOF
+done
+echo ""

.github/workflows/integrity.yml

@@ -0,0 +1,71 @@
+# Workflow to run basic integrity checks on OMT`s new Pull Requests and commits pushed into OMT repo
+
+name: OpenMapTiles Integrity CI
+
+on:
+  push:
+    branches: [ master, master-tools ]
+  pull_request:
+
+jobs:
+
+  integrity_test:
+    name: Run integrity test
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout the changes
+        uses: actions/checkout@v2
+
+      - name: Run quickstart for a small area
+        env:
+          area: monaco
+          QUIET: 1
+        run: |
+          echo MIN_ZOOM=0 >> .env
+          echo MAX_ZOOM=14 >> .env
+          ./quickstart.sh $area
+
+      - name: Save quickstart.log
+        uses: actions/upload-artifact@v2
+        with:
+          name: quickstart.log
+          path: quickstart.log
+
+      - name: Test etldoc images
+        run: |
+          export TEST_MODE=yes
+          make generate-devdoc
+
+      - name: Run quickstart and update in DIFF mode
+        env:
+          area: europe/monaco
+          QUIET: 1
+        run: |
+          echo MIN_ZOOM=0 >> .env
+          echo MAX_ZOOM=14 >> .env
+          echo DIFF_MODE=true >> .env
+          # Cleanup
+          rm -fr data build cache
+          # Create data/$area.repl.json
+          make download-geofabrik area=$area
+          # Download 2+ month old data
+          export old_date=$(date --date="$(date +%Y-%m-15) -2 month" +'%y%m01')
+          echo Downloading $old_date extract of $area
+          docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "wget -O data/$area.osm.pbf http://download.geofabrik.de/$area-$old_date.osm.pbf"
+          # Initial import and tile generation
+          ./quickstart.sh $area
+          sleep 2
+          echo Downloading updates
+          # Loop to recover from potential "ERROR 429: Too Many Requests"
+          docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "
+            while ! osmupdate --keep-tempfiles --base-url=$(sed -n 's/ *\"replication_url\": //p' data/$area.repl.json) data/$area.osm.pbf data/changes.osc.gz ; do 
+              sleep 2;
+              echo Sleeping...;
+              sleep 630;
+            done"
+          echo Downloading updates completed
+          echo Importing updates
+          make import-diff
+          echo Generating new tiles
+          make generate-tiles-pg