| 1 | #!/usr/bin/perl -w |
| 2 | |
| 3 | # OSM tile cleanup / maintenance script. Big parts are a copied from |
| 4 | # hochwasserloeschung.pl. |
| 5 | |
| 6 | use Fcntl ':mode'; |
| 7 | use POSIX 'strftime'; |
| 8 | |
| 9 | $zoommin = 0; |
| 10 | $zoommax = 20; |
| 11 | $filelimit = -1; |
| 12 | $rmemptydirs = 0; |
| 13 | $action = 0; |
| 14 | $rrs = '/bin/echo'; |
| 15 | $minage = 0; # In hours! |
| 16 | |
| 17 | # Sort function for candidate list |
| 18 | sub sortbydateasc { |
| 19 | #print("$a->[1] $b->[1]\n"); |
| 20 | return ($a->[1] <=> $b->[1]); |
| 21 | } |
| 22 | |
| 23 | # Remembers a candidate for deletion, if required sorting and purging out the |
| 24 | # list. |
| 25 | # Parameters: 0 filename |
| 26 | # 1 atime |
| 27 | # 2 mtime |
| 28 | # 3 type field from stat (containing DIR, LINK etc.) |
| 29 | # 4 uid |
| 30 | # 5 gid |
| 31 | # 6 size in bytes |
| 32 | sub remembercandidate($$$$$$$) { |
| 33 | if (($_[4] == 0) || ($_[5] == 0)) { |
| 34 | # we do not touch roots files or directories. |
| 35 | return; |
| 36 | } |
| 37 | if ($_[2] > (time() - ($minage * 3600))) { |
| 38 | #print("not adding $_[0], too new\n"); |
| 39 | return; |
| 40 | } |
| 41 | $totalfilesseen++; |
| 42 | push(@allentries, [ $_[0], $_[2], $_[3], $_[4], $_[5], $_[6] ]); |
| 43 | } |
| 44 | |
| 45 | # Par. 0: Path were we start |
| 46 | # Par. 1: Recursion counter |
| 47 | # Returns: Number of files/dirs found (non-recursively!) |
| 48 | sub runrecursive($$); |
| 49 | sub runrecursive($$) { |
| 50 | my $DIR; |
| 51 | my $pth = $_[0]; |
| 52 | my $curdirentry; |
| 53 | my @statres; |
| 54 | my $recctr = $_[1]; |
| 55 | my $direntries = 0; |
| 56 | unless (opendir($DIR, $pth)) { |
| 57 | print("ERROR: Failed to open dir $pth\n"); |
| 58 | exit(1); |
| 59 | } |
| 60 | while ($curdirentrys = readdir($DIR)) { |
| 61 | if (($curdirentrys eq '.') || ($curdirentrys eq '..')) { |
| 62 | next; |
| 63 | } |
| 64 | $direntries++; |
| 65 | $curdirentryf = $pth.'/'.$curdirentrys; |
| 66 | @statres = lstat($curdirentryf); |
| 67 | unless (@statres > 12) { |
| 68 | # This usually happens if we have a symlink to a file for which we have |
| 69 | # no permission to read. |
| 70 | #print("stat failed for $curdirentry\n"); |
| 71 | next; |
| 72 | } |
| 73 | if (S_ISLNK($statres[2])) { next; } # A symlink? We no like. |
| 74 | if ($recctr <= 4) { # There should be no files on these levels |
| 75 | unless (S_ISDIR($statres[2])) { next; } |
| 76 | unless ($curdirentrys =~ m/^\d+$/) { next; } |
| 77 | } else { # whereas there should be only files called .meta on level 5. |
| 78 | unless (S_ISREG($statres[2])) { next; } |
| 79 | unless ($curdirentrys =~ m/^\d+\.meta$/) { next; } |
| 80 | } |
| 81 | if ($recctr == 0) { |
| 82 | my $z = int($curdirentrys); |
| 83 | unless (($z >= $zoommin) && ($z <= $zoommax)) { next; } |
| 84 | runrecursive($curdirentryf, $recctr+1); |
| 85 | } elsif ($recctr <= 4) { |
| 86 | if (runrecursive($curdirentryf, $recctr+1) == 0) { # Empty dir! |
| 87 | if ($rmemptydirs) { |
| 88 | print("RMDIR: $curdirentryf\n"); |
| 89 | rmdir($curdirentryf); |
| 90 | } |
| 91 | } |
| 92 | } else { |
| 93 | remembercandidate($curdirentryf, $statres[8], $statres[9], $statres[2], $statres[4], $statres[5], $statres[12]); |
| 94 | } |
| 95 | } |
| 96 | closedir($DIR); |
| 97 | return $direntries; |
| 98 | } |
| 99 | |
| 100 | # Par. 0: Full path of file |
| 101 | sub rerenderfile($) { |
| 102 | my $fpth = $_[0]; |
| 103 | # We first need to figure out the relevant numbers from the full file path |
| 104 | unless ($fpth =~ m!/(\d+)/(\d+)/(\d+)/(\d+)/(\d+)/(\d+)\.meta$!) { |
| 105 | print("WARNING: rerenderfile: Failed to extract path components for rerendering out of '$fpth' - skipping\n"); |
| 106 | return; |
| 107 | } |
| 108 | my $zl = $1; my @p = ( $2, $3, $4, $5, $6 ); |
| 109 | my $calcx = 0; my $calcy = 0; |
| 110 | my $i; |
| 111 | for ($i = 0; $i < 5; $i++) { |
| 112 | $calcx = ($calcx << 4) | (($p[$i] & 0xf0) >> 4); |
| 113 | $calcy = ($calcy << 4) | (($p[$i] & 0x0f) >> 0); |
| 114 | } |
| 115 | # struct meta_layout { |
| 116 | # char magic[4]; // 'M' 'E' 'T' 'A' |
| 117 | # int count; // METATILE ^ 2 |
| 118 | # int x, y, z; // lowest x,y of this metatile, plus z |
| 119 | my $fi; |
| 120 | unless (open($fi, '<', $fpth)) { |
| 121 | print("WARNING: rerenderfile: Failed to open file '$fpth' - skipping\n"); |
| 122 | return; |
| 123 | } |
| 124 | binmode($fi); |
| 125 | my $dbuf; |
| 126 | unless (read($fi, $dbuf, 20)) { |
| 127 | print("WARNING: rerenderfile: Failed to read info from metatile '$fpth' - skipping\n"); |
| 128 | return; |
| 129 | } |
| 130 | close($fi); undef($fi); |
| 131 | unless (substr($dbuf, 0, 4) eq 'META') { |
| 132 | print("WARNING: rerenderfile: file '$fpth' is not a metatile - skipping\n"); |
| 133 | return; |
| 134 | } |
| 135 | unless (ord(substr($dbuf, 4, 1)) == 64) { |
| 136 | print("WARNING: rerenderfile: file '$fpth' is not a 8x8 metatile - skipping\n"); |
| 137 | return; |
| 138 | } |
| 139 | my $fx = (ord(substr($dbuf, 8, 1)) << 0) | (ord(substr($dbuf, 9, 1)) << 8) |
| 140 | | (ord(substr($dbuf, 10, 1)) << 16) | (ord(substr($dbuf, 11, 1)) << 24); |
| 141 | unless ($fx == $calcx) { |
| 142 | print("WARNING: rerenderfile: file '$fpth' is invalid - xsize $fx != $calcx - skipping\n"); |
| 143 | return; |
| 144 | } |
| 145 | my $fy = (ord(substr($dbuf, 12, 1)) << 0) | (ord(substr($dbuf, 13, 1)) << 8) |
| 146 | | (ord(substr($dbuf, 14, 1)) << 16) | (ord(substr($dbuf, 15, 1)) << 24); |
| 147 | unless ($fy == $calcy) { |
| 148 | print("WARNING: rerenderfile: file '$fpth' is invalid - xsize $fy != $calcy - skipping\n"); |
| 149 | return; |
| 150 | } |
| 151 | my $fz = (ord(substr($dbuf, 16, 1)) << 0) | (ord(substr($dbuf, 17, 1)) << 8) |
| 152 | | (ord(substr($dbuf, 18, 1)) << 16) | (ord(substr($dbuf, 19, 1)) << 24); |
| 153 | unless ($fz == $zl) { |
| 154 | print("WARNING: rerenderfile: file '$fpth' is invalid - zsize $fz != $zl - skipping\n"); |
| 155 | return; |
| 156 | } |
| 157 | print("Sending rendering request for z=$zl x=$calcx y=$calcy to regenerate '$fpth'\n"); |
| 158 | if (system("$rrs $zl $calcx $calcy")) { |
| 159 | print("Error executing $rrs $zl $calcx $calcy\n"); |
| 160 | } |
| 161 | } |
| 162 | |
| 163 | # Par. 0: x |
| 164 | # Par. 1: y |
| 165 | sub calcpathfromcomponents($$) { |
| 166 | my @res = (); |
| 167 | my $i; my $x = $_[0]; my $y = $_[1]; |
| 168 | for ($i = 4; $i >= 0; $i--) { |
| 169 | $res[$i] = sprintf("%d", (($x & 0x0f) << 4) + ($y & 0x0f)); |
| 170 | $x = $x >> 4; |
| 171 | $y = $y >> 4; |
| 172 | } |
| 173 | return $res[0] . "/" . $res[1] . "/" . $res[2] . "/" . $res[3] . "/" . $res[4]; |
| 174 | } |
| 175 | |
| 176 | sub dorerenderexpiredlist() { |
| 177 | my $ll; |
| 178 | %rerenderlist = (); |
| 179 | while ($ll = <STDIN>) { |
| 180 | if ($ll =~ m!^(\d+)/(\d+)/(\d+)$!) { |
| 181 | my $x = $2; my $y = $3; my $z = $1; |
| 182 | #print("Handling z=$z x=$x y=$y\n"); |
| 183 | if ($z != ($zoommax - 3)) { |
| 184 | print("Ignoring z=$z x=$x y=$y because of wrong zoom\n"); |
| 185 | next; |
| 186 | } |
| 187 | my $cz; my $cx; my $cy; my $curz; |
| 188 | for ($curz = $zoommin; $curz <= $zoommax; $curz++) { |
| 189 | $cz = $z; $cx = $x; $cy = $y; |
| 190 | while ($cz < $curz) { |
| 191 | $cz++; $cx <<= 1; $cy <<= 1; |
| 192 | } |
| 193 | while ($cz > $curz) { |
| 194 | $cz--; $cx >>= 1; $cy >>= 1; |
| 195 | } |
| 196 | #print("Matching tile at z=$cz: x=$cx y=$cy -"); |
| 197 | $cx = $cx & 0xfff8; $cy = $cy & 0xfff8; |
| 198 | #print(" rounded to x=$cx y=$cy\n"); |
| 199 | $rerenderlist{$cz}{$cx}{$cy} = 1; |
| 200 | } |
| 201 | } |
| 202 | } |
| 203 | $filesdone = 0; $filesseen = 0; |
| 204 | foreach $z (sort(keys(%rerenderlist))) { |
| 205 | foreach $x (sort(keys(%{$rerenderlist{$z}}))) { |
| 206 | foreach $y (sort(keys(%{$rerenderlist{$z}{$x}}))) { |
| 207 | my $p = ${fspath} . '/' . $z . '/' . calcpathfromcomponents($x, $y) . '.meta'; |
| 208 | #print("Checking: $z $x $y - Path: $p\n"); |
| 209 | $filesseen++; |
| 210 | if (-e $p) { |
| 211 | $filesdone++; |
| 212 | print("Sending rendering request for z=$z x=$x y=$y to regenerate '$p'\n"); |
| 213 | if (system("$rrs $z $x $y")) { |
| 214 | print("Error executing $rrs $z $x $y\n"); |
| 215 | } |
| 216 | } |
| 217 | } |
| 218 | } |
| 219 | } |
| 220 | print("Sent re-rendering-requests for $filesdone files that actually existed (of $filesseen candidates)\n"); |
| 221 | } |
| 222 | |
| 223 | $fspath = ''; |
| 224 | for ($i = 0; $i < @ARGV; $i++) { |
| 225 | $curarg = $ARGV[$i]; |
| 226 | if ($curarg eq '--help') { |
| 227 | print("Syntax: $0 [--help] [--zoom z] [--limit n] [--action a] [--rmemptydirs] [--rrs path] [--minage hrs] directory\n"); |
| 228 | print(" --zoom z[-z2]: Zoom level(s) to handle (default: 0-20).\n"); |
| 229 | print(" --limit n: Limit to the n oldest files (default: no limit)\n"); |
| 230 | print(" --action what: what action to perform with the found files.\n"); |
| 231 | print(" valid actions are: print delete rerender rerenderexpiredlist (default: print)\n"); |
| 232 | print(" rerenderexpiredlist: this is very different from all other commands, as it\n"); |
| 233 | print(" expects to read a list of expired tiles to rerender on STDIN. The list has\n"); |
| 234 | print(" to be in the format that osm2pgsql spits out.\n"); |
| 235 | print(" --rmemptydirs: remove empty subdirectories\n"); |
| 236 | print(" --rrs path: if action is rerender, this gives the path to the rerender script.\n"); |
| 237 | print(" it gets called with three parameters: z x y\n"); |
| 238 | print(" --minage hrs: only care about files that are at least hrs hours old\n"); |
| 239 | exit(1); |
| 240 | } elsif ($curarg eq '--zoom') { |
| 241 | $i++; |
| 242 | if ($i >= @ARGV) { |
| 243 | print("Error: --zoom requires a parameter\n"); |
| 244 | exit(1); |
| 245 | } else { |
| 246 | if ($ARGV[$i] =~ m/^(\d+)-(\d+)$/) { |
| 247 | $zoommin = $1; $zoommax = $2; |
| 248 | } elsif ($ARGV[$i] =~ m/^(\d+)$/) { |
| 249 | $zoommin = $1; $zoommax = $1; |
| 250 | } else { |
| 251 | print("Error: --zoom needs a single numeric parameter or a range (e.g. 4 or 10-12)\n"); |
| 252 | exit(1); |
| 253 | } |
| 254 | } |
| 255 | } elsif ($curarg eq '--limit') { |
| 256 | $i++; |
| 257 | if ($i >= @ARGV) { |
| 258 | print("Error: --limit requires a parameter\n"); |
| 259 | exit(1); |
| 260 | } else { |
| 261 | if ($ARGV[$i] =~ m/^(\d+)$/) { |
| 262 | $filelimit = $1; |
| 263 | } else { |
| 264 | print("Error: --limit requires an positive integer as parameter.\n"); |
| 265 | } |
| 266 | } |
| 267 | } elsif ($curarg eq '--action') { |
| 268 | $i++; |
| 269 | if ($i >= @ARGV) { |
| 270 | print("Error: --action requires a parameter\n"); |
| 271 | exit(1); |
| 272 | } else { |
| 273 | if ($ARGV[$i] eq 'print') { |
| 274 | $action = 0; |
| 275 | } elsif ($ARGV[$i] eq 'delete') { |
| 276 | $action = 1; |
| 277 | } elsif ($ARGV[$i] eq 'rerender') { |
| 278 | $action = 2; |
| 279 | } elsif ($ARGV[$i] eq 'rerenderexpiredlist') { |
| 280 | $action = 3; |
| 281 | } else { |
| 282 | print("Error: Invalid action selected.\n"); |
| 283 | exit(1); |
| 284 | } |
| 285 | } |
| 286 | } elsif ($curarg eq '--rrs') { |
| 287 | $i++; |
| 288 | if ($i >= @ARGV) { |
| 289 | print("Error: --rrs requires a parameter\n"); |
| 290 | exit(1); |
| 291 | } else { |
| 292 | $rrs = $ARGV[$i]; |
| 293 | } |
| 294 | } elsif ($curarg eq '--minage') { |
| 295 | $i++; |
| 296 | if ($i >= @ARGV) { |
| 297 | print("Error: --minage requires a parameter\n"); |
| 298 | exit(1); |
| 299 | } else { |
| 300 | if ($ARGV[$i] =~ m/^(\d+)$/) { |
| 301 | $minage = $1; |
| 302 | } else { |
| 303 | print("Error: --minage requires an positive integer as parameter.\n"); |
| 304 | } |
| 305 | } |
| 306 | } elsif ($curarg eq '--rmemptydirs') { |
| 307 | $rmemptydirs = 1; |
| 308 | } else { |
| 309 | if ($fspath eq '') { |
| 310 | $fspath = $curarg; |
| 311 | } else { |
| 312 | print("Too many parameters, or parameter(s) not understood.\n"); |
| 313 | print("I can only handle one directory parameter, but I consider both "); |
| 314 | print("'$curarg' and '$fspath' a pathname since they are not a known parameter.\n"); |
| 315 | exit(1); |
| 316 | } |
| 317 | } |
| 318 | } |
| 319 | if ($fspath eq '') { |
| 320 | print("ERROR: No path to clear given.\n"); |
| 321 | exit(1); |
| 322 | } |
| 323 | if ($action == 3) { # This significantly differs from the rest of our operations |
| 324 | dorerenderexpiredlist(); |
| 325 | exit(0); |
| 326 | } |
| 327 | @allentries = (); |
| 328 | $totalfilesseen = 0; |
| 329 | runrecursive($fspath, 0); |
| 330 | @allentries = sort(sortbydateasc @allentries); |
| 331 | print(int(@allentries)." files seen\n"); |
| 332 | $filesdone = 0; |
| 333 | foreach $ent (@allentries) { |
| 334 | if ($filelimit > 0) { |
| 335 | if ($filesdone >= $filelimit) { |
| 336 | print("File limit $filelimit reached, exiting.\n"); |
| 337 | exit(0); |
| 338 | } |
| 339 | } |
| 340 | #print("Handling File '$ent->[0]', $ent->[5] blocks [u: $ent->[3] g: $ent->[4] mtime: ".strftime("%Y-%m-%d.%H:%M:%S", localtime($ent->[1]))."]\n"); |
| 341 | if ($action == 0) { # Print |
| 342 | print("$ent->[0]\n"); |
| 343 | } elsif ($action == 1) { # Delete |
| 344 | print("DELETING $ent->[0] (mtime " . strftime("%Y-%m-%d.%H:%M:%S", localtime($ent->[1])) . ")\n"); |
| 345 | unless (unlink($ent->[0])) { |
| 346 | print("ERROR: rm for $ent->[0] failed!\n"); |
| 347 | } |
| 348 | } elsif ($action == 2) { # Rerender |
| 349 | rerenderfile($ent->[0]); |
| 350 | } |
| 351 | $filesdone++; |
| 352 | } |