From 5203e8e5f81d512c3365deef8cfb4df359e1ce8e Mon Sep 17 00:00:00 2001 From: User for running Openstreetmap-things Date: Sun, 15 Jul 2012 23:47:19 +0200 Subject: [PATCH] these two are the cronjobs-scripts --- scripts/dailymaint.sh | 32 ++++++++++++++++ scripts/doosmupdate.sh | 84 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100755 scripts/dailymaint.sh create mode 100755 scripts/doosmupdate.sh diff --git a/scripts/dailymaint.sh b/scripts/dailymaint.sh new file mode 100755 index 0000000..f321130 --- /dev/null +++ b/scripts/dailymaint.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +echo "Daily maintenance starting: `date`" +# Delete tiles in zoomlevel 13-14 that are older than 336 hours = 14 days +sudo -u tirex -- /home.local/osm/bin/osmtilecleanup.pl --zoom 13-14 --action delete --rmemptydirs --minage 336 /var/lib/tirex/tiles/osm +sudo -u tirex -- /home.local/osm/bin/osmtilecleanup.pl --zoom 13-14 --action delete --rmemptydirs --minage 336 /var/lib/tirex/tiles/osmde +# Delete tiles in zoomlevel 15-20 that are older than 168 hours = 7 days +sudo -u tirex -- /home.local/osm/bin/osmtilecleanup.pl --zoom 15-20 --action delete --rmemptydirs --minage 168 /var/lib/tirex/tiles/osm +sudo -u tirex -- /home.local/osm/bin/osmtilecleanup.pl --zoom 15-20 --action delete --rmemptydirs --minage 168 /var/lib/tirex/tiles/osmde +# Regenerate a part of zoomlevel 0-12 tiles. There are at most 350000 such tiles. +# At 5000 a day, it'll take 70 days to regenerate all. As they hardly ever change, that should be fine. +/home.local/osm/bin/osmtilecleanup.pl --zoom 0-12 --action rerender \ + --limit 5000 --rrs /home.local/osm/bin/osmrerenderwrapper.sh /var/lib/tirex/tiles/osm +/home.local/osm/bin/osmtilecleanup.pl --zoom 0-12 --action rerender \ + --limit 5000 --rrs /home.local/osm/bin/osmrerenderwrapper-osmde.sh /var/lib/tirex/tiles/osmde +# Update tile stats for munin +tirex-tiledir-check --stats=/var/lib/tirex/stats/tiles-osm.stats osm +tirex-tiledir-check --stats=/var/lib/tirex/stats/tiles-osmde.stats osmde + +DOW=`/bin/date '+%w'` +if [ "$DOW" == "6" -o "$DOW" == "3" ] ; then + echo "Time to vacuum the databse... This will take a few hours." + # Vacuum the database. This can take hours. Possibly better just once per week? + time psql --dbname=osm -c 'vacuum verbose analyze' +fi + +# Rerender Erlangen / Nuernberg. Around 4000 tiles, takes around 30 minutes. +tirex-batch --prio=22 --expire=+14400 map=osm bbox=10.93,49.37,11.14,49.63 z=13-19 +tirex-batch --prio=22 --expire=+14400 map=osmde bbox=10.93,49.37,11.14,49.63 z=13-19 + +echo "Daily maintenance finished: `date`" + diff --git a/scripts/doosmupdate.sh b/scripts/doosmupdate.sh new file mode 100755 index 0000000..53f63c6 --- /dev/null +++ b/scripts/doosmupdate.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +LOCKFILE=/home.local/osm/run/.osmupdate.lock +CHANGESFILE=/home.local/osm/run/osmosis-changes.osc.gz +EXPIRYFILE=/home.local/osm/run/expiredtiles.lst +OSMOSISWORKDIR=/home.local/osm/.osmosis +WEBSTATEFILE=/var/www/lastdbupdate.txt +# Note: set the expire-tiles to 3 less than the real maximum, +# as this makes the tilelist MUCH shorter, and the tiles get rerendered +# anyways due to the metatile-layout! +UPDATECMD="/usr/bin/osm2pgsql --slim --cache=8000 --append --database=osm \ + --style=/home.local/osm/dbstyle/osm2pgsql-new.style \ + --hstore --expire-tiles=17-17 \ + --expire-output=$EXPIRYFILE $CHANGESFILE" + +cleanup() +{ + rm -f "$LOCKFILE" + exit 1 +} + +trytofeedupdate() +{ + echo "Trying to feed updates into databse..." + echo "Executing: $UPDATECMD" + $UPDATECMD + if [ $? == 0 ] ; then + echo "Update went fine, removing changes-file" + rm -f $CHANGESFILE + else + echo "Update failed with error $?" + fi +} + +echo "Starting update script: `date`" +if [ -e "$LOCKFILE" ] ; then + echo "Lockfile exists - ABORTING" + exit 1 +fi +touch "$LOCKFILE" || exit 9 +trap cleanup 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 \ + 20 21 22 23 24 25 26 27 28 29 30 31 +if [ -e $CHANGESFILE ] ; then + echo "Changesfile exists - will try to feed update to Database" + trytofeedupdate +fi +if [ -e $CHANGESFILE ] ; then + echo "Changesfile STILL exists - ABORTING" + cleanup + exit 1 +fi +echo "Downloading changes with OSMOSIS..." +echo "Executing: /usr/bin/osmosis --read-replication-interval workingDirectory=$OSMOSISWORKDIR --simplify-change --write-xml-change $CHANGESFILE" +/usr/bin/osmosis --read-replication-interval workingDirectory=$OSMOSISWORKDIR --simplify-change --write-xml-change $CHANGESFILE +if [ $? != 0 ] ; then + echo "Osmosis execution failed. ABORTING" + cleanup + exit 1 +fi +trytofeedupdate +if [ -e $CHANGESFILE ] ; then + echo "Changesfile STILL exists - update must have failed. ABORTING" + cleanup + exit 1 +fi +echo "Update seems to have been successful." +grep timestamp $OSMOSISWORKDIR/state.txt | sed -e 's/timestamp=//g' -e 's/T/ /g' -e 's/\\:/:/g' > $WEBSTATEFILE +if [ -e $EXPIRYFILE ] ; then + echo "Expiryfile exists. Telling tirex to rerender the tiles mentioned there..." + echo "Number of lines: `wc -l $EXPIRYFILE`" + cat $EXPIRYFILE | /home.local/osm/bin/osmtilecleanup.pl \ + --action rerenderexpiredlist --zoom 13-20 \ + --rrs /home.local/osm/bin/osmrerenderwrapper.sh \ + /var/lib/tirex/tiles/osm + cat $EXPIRYFILE | /home.local/osm/bin/osmtilecleanup.pl \ + --action rerenderexpiredlist --zoom 13-20 \ + --rrs /home.local/osm/bin/osmrerenderwrapper.sh \ + /var/lib/tirex/tiles/osmde + rm $EXPIRYFILE +fi +echo "Ending update script cleanly: `date`" +cleanup +exit 0 + -- 2.25.1