3 # OSM tile cleanup / maintenance script. Big parts are a copied from
4 # hochwasserloeschung.pl.
15 $minage = 0; # In hours!
16 $setbackinterval = 7305 * 86400; # 20 years (including leap years)
18 # Sort function for candidate list
20 #print("$a->[1] $b->[1]\n");
21 return ($a->[1] <=> $b->[1]);
24 # Remembers a candidate for deletion, if required sorting and purging out the
26 # Parameters: 0 filename
29 # 3 type field from stat (containing DIR, LINK etc.)
33 sub remembercandidate($$$$$$$) {
34 if (($_[4] == 0) || ($_[5] == 0)) {
35 # we do not touch roots files or directories.
38 if ($_[2] > (time() - ($minage * 3600))) {
39 #print("not adding $_[0], too new\n");
43 push(@allentries, [ $_[0], $_[2], $_[3], $_[4], $_[5], $_[6] ]);
46 # Par. 0: Path were we start
47 # Par. 1: Recursion counter
48 # Returns: Number of files/dirs found (non-recursively!)
50 sub runrecursive($$) {
57 unless (opendir($DIR, $pth)) {
58 print("ERROR: Failed to open dir $pth\n");
61 while ($curdirentrys = readdir($DIR)) {
62 if (($curdirentrys eq '.') || ($curdirentrys eq '..')) {
66 $curdirentryf = $pth.'/'.$curdirentrys;
67 @statres = lstat($curdirentryf);
68 unless (@statres > 12) {
69 # This usually happens if we have a symlink to a file for which we have
70 # no permission to read.
71 #print("stat failed for $curdirentry\n");
74 if (S_ISLNK($statres[2])) { next; } # A symlink? We no like.
75 if ($recctr <= 4) { # There should be no files on these levels
76 unless (S_ISDIR($statres[2])) { next; }
77 unless ($curdirentrys =~ m/^\d+$/) { next; }
78 } else { # whereas there should be only files called .meta on level 5.
79 unless (S_ISREG($statres[2])) { next; }
80 unless ($curdirentrys =~ m/^\d+\.meta$/) { next; }
83 my $z = int($curdirentrys);
84 unless (($z >= $zoommin) && ($z <= $zoommax)) { next; }
85 runrecursive($curdirentryf, $recctr+1);
86 } elsif ($recctr <= 4) {
87 if (runrecursive($curdirentryf, $recctr+1) == 0) { # Empty dir!
89 print("RMDIR: $curdirentryf\n");
94 remembercandidate($curdirentryf, $statres[8], $statres[9], $statres[2], $statres[4], $statres[5], $statres[12]);
101 # Par. 0: Full path of file
102 sub rerenderfile($) {
104 # We first need to figure out the relevant numbers from the full file path
105 unless ($fpth =~ m!/(\d+)/(\d+)/(\d+)/(\d+)/(\d+)/(\d+)\.meta$!) {
106 print("WARNING: rerenderfile: Failed to extract path components for rerendering out of '$fpth' - skipping\n");
109 my $zl = $1; my @p = ( $2, $3, $4, $5, $6 );
110 my $calcx = 0; my $calcy = 0;
112 for ($i = 0; $i < 5; $i++) {
113 $calcx = ($calcx << 4) | (($p[$i] & 0xf0) >> 4);
114 $calcy = ($calcy << 4) | (($p[$i] & 0x0f) >> 0);
116 # struct meta_layout {
117 # char magic[4]; // 'M' 'E' 'T' 'A'
118 # int count; // METATILE ^ 2
119 # int x, y, z; // lowest x,y of this metatile, plus z
121 unless (open($fi, '<', $fpth)) {
122 print("WARNING: rerenderfile: Failed to open file '$fpth' - skipping\n");
127 unless (read($fi, $dbuf, 20)) {
128 print("WARNING: rerenderfile: Failed to read info from metatile '$fpth' - skipping\n");
131 close($fi); undef($fi);
132 unless (substr($dbuf, 0, 4) eq 'META') {
133 print("WARNING: rerenderfile: file '$fpth' is not a metatile - skipping\n");
136 unless (ord(substr($dbuf, 4, 1)) == 64) {
137 print("WARNING: rerenderfile: file '$fpth' is not a 8x8 metatile - skipping\n");
140 my $fx = (ord(substr($dbuf, 8, 1)) << 0) | (ord(substr($dbuf, 9, 1)) << 8)
141 | (ord(substr($dbuf, 10, 1)) << 16) | (ord(substr($dbuf, 11, 1)) << 24);
142 unless ($fx == $calcx) {
143 print("WARNING: rerenderfile: file '$fpth' is invalid - xsize $fx != $calcx - skipping\n");
146 my $fy = (ord(substr($dbuf, 12, 1)) << 0) | (ord(substr($dbuf, 13, 1)) << 8)
147 | (ord(substr($dbuf, 14, 1)) << 16) | (ord(substr($dbuf, 15, 1)) << 24);
148 unless ($fy == $calcy) {
149 print("WARNING: rerenderfile: file '$fpth' is invalid - xsize $fy != $calcy - skipping\n");
152 my $fz = (ord(substr($dbuf, 16, 1)) << 0) | (ord(substr($dbuf, 17, 1)) << 8)
153 | (ord(substr($dbuf, 18, 1)) << 16) | (ord(substr($dbuf, 19, 1)) << 24);
154 unless ($fz == $zl) {
155 print("WARNING: rerenderfile: file '$fpth' is invalid - zsize $fz != $zl - skipping\n");
158 print("Sending rendering request for z=$zl x=$calcx y=$calcy to regenerate '$fpth'\n");
159 if (system("$rrs $zl $calcx $calcy")) {
160 print("Error executing $rrs $zl $calcx $calcy\n");
166 sub calcpathfromcomponents($$) {
168 my $i; my $x = $_[0]; my $y = $_[1];
169 for ($i = 4; $i >= 0; $i--) {
170 $res[$i] = sprintf("%d", (($x & 0x0f) << 4) + ($y & 0x0f));
174 return $res[0] . "/" . $res[1] . "/" . $res[2] . "/" . $res[3] . "/" . $res[4];
177 sub dohandleexpiredlist() {
180 while ($ll = <STDIN>) {
181 if ($ll =~ m!^(\d+)/(\d+)/(\d+)$!) {
182 my $x = $2; my $y = $3; my $z = $1;
183 #print("Handling z=$z x=$x y=$y\n");
184 if ($z != ($zoommax - 3)) {
185 print("Ignoring z=$z x=$x y=$y because of wrong zoom\n");
188 my $cz; my $cx; my $cy; my $curz;
189 for ($curz = $zoommin; $curz <= $zoommax; $curz++) {
190 $cz = $z; $cx = $x; $cy = $y;
191 while ($cz < $curz) {
192 $cz++; $cx <<= 1; $cy <<= 1;
194 while ($cz > $curz) {
195 $cz--; $cx >>= 1; $cy >>= 1;
197 #print("Matching tile at z=$cz: x=$cx y=$cy -");
198 $cx = $cx & 0xfff8; $cy = $cy & 0xfff8;
199 #print(" rounded to x=$cx y=$cy\n");
200 $rerenderlist{$cz}{$cx}{$cy} = 1;
204 $filesdone = 0; $filesseen = 0;
205 foreach $z (sort(keys(%rerenderlist))) {
206 foreach $x (sort(keys(%{$rerenderlist{$z}}))) {
207 foreach $y (sort(keys(%{$rerenderlist{$z}{$x}}))) {
208 my $p = ${fspath} . '/' . $z . '/' . calcpathfromcomponents($x, $y) . '.meta';
209 #print("Checking: $z $x $y - Path: $p\n");
214 print("Sending rendering request for z=$z x=$x y=$y to regenerate '$p'\n");
215 if (system("$rrs $z $x $y")) {
216 print("Error executing $rrs $z $x $y\n");
218 } elsif ($action == 4) {
219 my $mtime = (stat($p))[9];
220 my $curtime = time();
221 if (($curtime - $setbackinterval) > $mtime) {
222 # Do not touch again - it's already 20 years back.
223 print("Not touching '$p', it's over 20 years old so probably has already been set back.\n");
225 my $newmtime = $mtime - $setbackinterval;
226 print("Touching '$p' (z=$z x=$x y=$y)\n");
227 if (utime($curtime, $newmtime, $p) < 1) {
228 print("Error touching '$p': $!\n");
232 print("Internal error - action variable invalid. This is a programming error.\n");
240 print("Sent re-rendering-requests for $filesdone files that actually existed (of $filesseen candidates)\n");
241 } elsif ($action == 4) {
242 print("Touched $filesdone files (of $filesseen candidates)\n");
247 for ($i = 0; $i < @ARGV; $i++) {
249 if ($curarg eq '--help') {
250 print("Syntax: $0 [--help] [--zoom z] [--limit n] [--action a] [--rmemptydirs]\n");
251 print(" [--rrs path] [--minage hrs] directory\n");
252 print(" --zoom z[-z2]: Zoom level(s) to handle (default: 0-20).\n");
253 print(" --limit n: Limit to the n oldest files (default: no limit)\n");
254 print(" --action what: what action to perform with the found files.\n");
255 print(" valid actions are: print delete rerender rerenderexpiredlist\n");
256 print(" touchexpiredlist (default: print)\n");
257 print(" rerenderexpiredlist and touchexpiredlist: these are very different from the\n");
258 print(" other commands, as they expect to read a list of expired tiles to rerender\n");
259 print(" or touch on STDIN. The list has to be in the format that osm2pgsql spits\n");
260 print(" out. touchexpiredlist sets the mtime of the tiles back 20 years.\n");
261 print(" --rmemptydirs: remove empty subdirectories\n");
262 print(" --rrs path: if action is rerender, this gives the path to the rerender script.\n");
263 print(" it gets called with three parameters: z x y\n");
264 print(" --minage hrs: only care about files that are at least hrs hours old\n");
266 } elsif ($curarg eq '--zoom') {
269 print("Error: --zoom requires a parameter\n");
272 if ($ARGV[$i] =~ m/^(\d+)-(\d+)$/) {
273 $zoommin = $1; $zoommax = $2;
274 } elsif ($ARGV[$i] =~ m/^(\d+)$/) {
275 $zoommin = $1; $zoommax = $1;
277 print("Error: --zoom needs a single numeric parameter or a range (e.g. 4 or 10-12)\n");
281 } elsif ($curarg eq '--limit') {
284 print("Error: --limit requires a parameter\n");
287 if ($ARGV[$i] =~ m/^(\d+)$/) {
290 print("Error: --limit requires an positive integer as parameter.\n");
293 } elsif ($curarg eq '--action') {
296 print("Error: --action requires a parameter\n");
299 if ($ARGV[$i] eq 'print') {
301 } elsif ($ARGV[$i] eq 'delete') {
303 } elsif ($ARGV[$i] eq 'rerender') {
305 } elsif ($ARGV[$i] eq 'rerenderexpiredlist') {
307 } elsif ($ARGV[$i] eq 'touchexpiredlist') {
310 print("Error: Invalid action selected.\n");
314 } elsif ($curarg eq '--rrs') {
317 print("Error: --rrs requires a parameter\n");
322 } elsif ($curarg eq '--minage') {
325 print("Error: --minage requires a parameter\n");
328 if ($ARGV[$i] =~ m/^(\d+)$/) {
331 print("Error: --minage requires an positive integer as parameter.\n");
334 } elsif ($curarg eq '--rmemptydirs') {
340 print("Too many parameters, or parameter(s) not understood.\n");
341 print("I can only handle one directory parameter, but I consider both ");
342 print("'$curarg' and '$fspath' a pathname since they are not a known parameter.\n");
348 print("ERROR: No path to clear given.\n");
351 if (($action == 3) || ($action == 4)) { # This significantly differs from the rest of our operations
352 dohandleexpiredlist();
357 runrecursive($fspath, 0);
358 @allentries = sort(sortbydateasc @allentries);
359 print(int(@allentries)." files seen\n");
361 foreach $ent (@allentries) {
362 if ($filelimit > 0) {
363 if ($filesdone >= $filelimit) {
364 print("File limit $filelimit reached, exiting.\n");
368 #print("Handling File '$ent->[0]', $ent->[5] blocks [u: $ent->[3] g: $ent->[4] mtime: ".strftime("%Y-%m-%d.%H:%M:%S", localtime($ent->[1]))."]\n");
369 if ($action == 0) { # Print
370 print("$ent->[0]\n");
371 } elsif ($action == 1) { # Delete
372 print("DELETING $ent->[0] (mtime " . strftime("%Y-%m-%d.%H:%M:%S", localtime($ent->[1])) . ")\n");
373 unless (unlink($ent->[0])) {
374 print("ERROR: rm for $ent->[0] failed!\n");
376 } elsif ($action == 2) { # Rerender
377 rerenderfile($ent->[0]);