3 # OSM tile cleanup / maintenance script. Big parts are a copied from
4 # hochwasserloeschung.pl.
15 $minage = 0; # In hours!
17 # Sort function for candidate list
19 #print("$a->[1] $b->[1]\n");
20 return ($a->[1] <=> $b->[1]);
23 # Remembers a candidate for deletion, if required sorting and purging out the
25 # Parameters: 0 filename
28 # 3 type field from stat (containing DIR, LINK etc.)
32 sub remembercandidate($$$$$$$) {
33 if (($_[4] == 0) || ($_[5] == 0)) {
34 # we do not touch roots files or directories.
37 if ($_[2] > (time() - ($minage * 3600))) {
38 #print("not adding $_[0], too new\n");
42 push(@allentries, [ $_[0], $_[2], $_[3], $_[4], $_[5], $_[6] ]);
45 # Par. 0: Path were we start
46 # Par. 1: Recursion counter
47 # Returns: Number of files/dirs found (non-recursively!)
49 sub runrecursive($$) {
56 unless (opendir($DIR, $pth)) {
57 print("ERROR: Failed to open dir $pth\n");
60 while ($curdirentrys = readdir($DIR)) {
61 if (($curdirentrys eq '.') || ($curdirentrys eq '..')) {
65 $curdirentryf = $pth.'/'.$curdirentrys;
66 @statres = lstat($curdirentryf);
67 unless (@statres > 12) {
68 # This usually happens if we have a symlink to a file for which we have
69 # no permission to read.
70 #print("stat failed for $curdirentry\n");
73 if (S_ISLNK($statres[2])) { next; } # A symlink? We no like.
74 if ($recctr <= 4) { # There should be no files on these levels
75 unless (S_ISDIR($statres[2])) { next; }
76 unless ($curdirentrys =~ m/^\d+$/) { next; }
77 } else { # whereas there should be only files called .meta on level 5.
78 unless (S_ISREG($statres[2])) { next; }
79 unless ($curdirentrys =~ m/^\d+\.meta$/) { next; }
82 my $z = int($curdirentrys);
83 unless (($z >= $zoommin) && ($z <= $zoommax)) { next; }
84 runrecursive($curdirentryf, $recctr+1);
85 } elsif ($recctr <= 4) {
86 if (runrecursive($curdirentryf, $recctr+1) == 0) { # Empty dir!
88 print("RMDIR: $curdirentryf\n");
93 remembercandidate($curdirentryf, $statres[8], $statres[9], $statres[2], $statres[4], $statres[5], $statres[12]);
100 # Par. 0: Full path of file
101 sub rerenderfile($) {
103 # We first need to figure out the relevant numbers from the full file path
104 unless ($fpth =~ m!/(\d+)/(\d+)/(\d+)/(\d+)/(\d+)/(\d+)\.meta$!) {
105 print("WARNING: rerenderfile: Failed to extract path components for rerendering out of '$fpth' - skipping\n");
108 my $zl = $1; my @p = ( $2, $3, $4, $5, $6 );
109 my $calcx = 0; my $calcy = 0;
111 for ($i = 0; $i < 5; $i++) {
112 $calcx = ($calcx << 4) | (($p[$i] & 0xf0) >> 4);
113 $calcy = ($calcy << 4) | (($p[$i] & 0x0f) >> 0);
115 # struct meta_layout {
116 # char magic[4]; // 'M' 'E' 'T' 'A'
117 # int count; // METATILE ^ 2
118 # int x, y, z; // lowest x,y of this metatile, plus z
120 unless (open($fi, '<', $fpth)) {
121 print("WARNING: rerenderfile: Failed to open file '$fpth' - skipping\n");
126 unless (read($fi, $dbuf, 20)) {
127 print("WARNING: rerenderfile: Failed to read info from metatile '$fpth' - skipping\n");
130 close($fi); undef($fi);
131 unless (substr($dbuf, 0, 4) eq 'META') {
132 print("WARNING: rerenderfile: file '$fpth' is not a metatile - skipping\n");
135 unless (ord(substr($dbuf, 4, 1)) == 64) {
136 print("WARNING: rerenderfile: file '$fpth' is not a 8x8 metatile - skipping\n");
139 my $fx = (ord(substr($dbuf, 8, 1)) << 0) | (ord(substr($dbuf, 9, 1)) << 8)
140 | (ord(substr($dbuf, 10, 1)) << 16) | (ord(substr($dbuf, 11, 1)) << 24);
141 unless ($fx == $calcx) {
142 print("WARNING: rerenderfile: file '$fpth' is invalid - xsize $fx != $calcx - skipping\n");
145 my $fy = (ord(substr($dbuf, 12, 1)) << 0) | (ord(substr($dbuf, 13, 1)) << 8)
146 | (ord(substr($dbuf, 14, 1)) << 16) | (ord(substr($dbuf, 15, 1)) << 24);
147 unless ($fy == $calcy) {
148 print("WARNING: rerenderfile: file '$fpth' is invalid - xsize $fy != $calcy - skipping\n");
151 my $fz = (ord(substr($dbuf, 16, 1)) << 0) | (ord(substr($dbuf, 17, 1)) << 8)
152 | (ord(substr($dbuf, 18, 1)) << 16) | (ord(substr($dbuf, 19, 1)) << 24);
153 unless ($fz == $zl) {
154 print("WARNING: rerenderfile: file '$fpth' is invalid - zsize $fz != $zl - skipping\n");
157 print("Sending rendering request for z=$zl x=$calcx y=$calcy to regenerate '$fpth'\n");
158 if (system("$rrs $zl $calcx $calcy")) {
159 print("Error executing $rrs $zl $calcx $calcy\n");
165 sub calcpathfromcomponents($$) {
167 my $i; my $x = $_[0]; my $y = $_[1];
168 for ($i = 4; $i >= 0; $i--) {
169 $res[$i] = sprintf("%d", (($x & 0x0f) << 4) + ($y & 0x0f));
173 return $res[0] . "/" . $res[1] . "/" . $res[2] . "/" . $res[3] . "/" . $res[4];
176 sub dorerenderexpiredlist() {
179 while ($ll = <STDIN>) {
180 if ($ll =~ m!^(\d+)/(\d+)/(\d+)$!) {
181 my $x = $2; my $y = $3; my $z = $1;
182 #print("Handling z=$z x=$x y=$y\n");
183 if ($z != ($zoommax - 3)) {
184 print("Ignoring z=$z x=$x y=$y because of wrong zoom\n");
187 my $cz; my $cx; my $cy; my $curz;
188 for ($curz = $zoommin; $curz <= $zoommax; $curz++) {
189 $cz = $z; $cx = $x; $cy = $y;
190 while ($cz < $curz) {
191 $cz++; $cx <<= 1; $cy <<= 1;
193 while ($cz > $curz) {
194 $cz--; $cx >>= 1; $cy >>= 1;
196 #print("Matching tile at z=$cz: x=$cx y=$cy -");
197 $cx = $cx & 0xfff8; $cy = $cy & 0xfff8;
198 #print(" rounded to x=$cx y=$cy\n");
199 $rerenderlist{$cz}{$cx}{$cy} = 1;
203 $filesdone = 0; $filesseen = 0;
204 foreach $z (sort(keys(%rerenderlist))) {
205 foreach $x (sort(keys(%{$rerenderlist{$z}}))) {
206 foreach $y (sort(keys(%{$rerenderlist{$z}{$x}}))) {
207 my $p = ${fspath} . '/' . $z . '/' . calcpathfromcomponents($x, $y) . '.meta';
208 #print("Checking: $z $x $y - Path: $p\n");
212 print("Sending rendering request for z=$z x=$x y=$y to regenerate '$p'\n");
213 if (system("$rrs $z $x $y")) {
214 print("Error executing $rrs $z $x $y\n");
220 print("Sent re-rendering-requests for $filesdone files that actually existed (of $filesseen candidates)\n");
224 for ($i = 0; $i < @ARGV; $i++) {
226 if ($curarg eq '--help') {
227 print("Syntax: $0 [--help] [--zoom z] [--limit n] [--action a] [--rmemptydirs] [--rrs path] [--minage hrs] directory\n");
228 print(" --zoom z[-z2]: Zoom level(s) to handle (default: 0-20).\n");
229 print(" --limit n: Limit to the n oldest files (default: no limit)\n");
230 print(" --action what: what action to perform with the found files.\n");
231 print(" valid actions are: print delete rerender rerenderexpiredlist (default: print)\n");
232 print(" rerenderexpiredlist: this is very different from all other commands, as it\n");
233 print(" expects to read a list of expired tiles to rerender on STDIN. The list has\n");
234 print(" to be in the format that osm2pgsql spits out.\n");
235 print(" --rmemptydirs: remove empty subdirectories\n");
236 print(" --rrs path: if action is rerender, this gives the path to the rerender script.\n");
237 print(" it gets called with three parameters: z x y\n");
238 print(" --minage hrs: only care about files that are at least hrs hours old\n");
240 } elsif ($curarg eq '--zoom') {
243 print("Error: --zoom requires a parameter\n");
246 if ($ARGV[$i] =~ m/^(\d+)-(\d+)$/) {
247 $zoommin = $1; $zoommax = $2;
248 } elsif ($ARGV[$i] =~ m/^(\d+)$/) {
249 $zoommin = $1; $zoommax = $1;
251 print("Error: --zoom needs a single numeric parameter or a range (e.g. 4 or 10-12)\n");
255 } elsif ($curarg eq '--limit') {
258 print("Error: --limit requires a parameter\n");
261 if ($ARGV[$i] =~ m/^(\d+)$/) {
264 print("Error: --limit requires an positive integer as parameter.\n");
267 } elsif ($curarg eq '--action') {
270 print("Error: --action requires a parameter\n");
273 if ($ARGV[$i] eq 'print') {
275 } elsif ($ARGV[$i] eq 'delete') {
277 } elsif ($ARGV[$i] eq 'rerender') {
279 } elsif ($ARGV[$i] eq 'rerenderexpiredlist') {
282 print("Error: Invalid action selected.\n");
286 } elsif ($curarg eq '--rrs') {
289 print("Error: --rrs requires a parameter\n");
294 } elsif ($curarg eq '--minage') {
297 print("Error: --minage requires a parameter\n");
300 if ($ARGV[$i] =~ m/^(\d+)$/) {
303 print("Error: --minage requires an positive integer as parameter.\n");
306 } elsif ($curarg eq '--rmemptydirs') {
312 print("Too many parameters, or parameter(s) not understood.\n");
313 print("I can only handle one directory parameter, but I consider both ");
314 print("'$curarg' and '$fspath' a pathname since they are not a known parameter.\n");
320 print("ERROR: No path to clear given.\n");
323 if ($action == 3) { # This significantly differs from the rest of our operations
324 dorerenderexpiredlist();
329 runrecursive($fspath, 0);
330 @allentries = sort(sortbydateasc @allentries);
331 print(int(@allentries)." files seen\n");
333 foreach $ent (@allentries) {
334 if ($filelimit > 0) {
335 if ($filesdone >= $filelimit) {
336 print("File limit $filelimit reached, exiting.\n");
340 #print("Handling File '$ent->[0]', $ent->[5] blocks [u: $ent->[3] g: $ent->[4] mtime: ".strftime("%Y-%m-%d.%H:%M:%S", localtime($ent->[1]))."]\n");
341 if ($action == 0) { # Print
342 print("$ent->[0]\n");
343 } elsif ($action == 1) { # Delete
344 print("DELETING $ent->[0] (mtime " . strftime("%Y-%m-%d.%H:%M:%S", localtime($ent->[1])) . ")\n");
345 unless (unlink($ent->[0])) {
346 print("ERROR: rm for $ent->[0] failed!\n");
348 } elsif ($action == 2) { # Rerender
349 rerenderfile($ent->[0]);