4b713169 |
1 | #!/usr/bin/perl -w |
2 | |
3 | # OSM tile cleanup / maintenance script. Big parts are a copied from |
4 | # hochwasserloeschung.pl. |
5 | |
6 | use Fcntl ':mode'; |
7 | use POSIX 'strftime'; |
8 | |
9 | $zoommin = 0; |
10 | $zoommax = 20; |
11 | $filelimit = -1; |
12 | $rmemptydirs = 0; |
13 | $action = 0; |
14 | $rrs = '/bin/echo'; |
15 | $minage = 0; # In hours! |
16 | |
17 | # Sort function for candidate list |
18 | sub sortbydateasc { |
19 | #print("$a->[1] $b->[1]\n"); |
20 | return ($a->[1] <=> $b->[1]); |
21 | } |
22 | |
23 | # Remembers a candidate for deletion, if required sorting and purging out the |
24 | # list. |
25 | # Parameters: 0 filename |
26 | # 1 atime |
27 | # 2 mtime |
28 | # 3 type field from stat (containing DIR, LINK etc.) |
29 | # 4 uid |
30 | # 5 gid |
31 | # 6 size in bytes |
32 | sub remembercandidate($$$$$$$) { |
33 | if (($_[4] == 0) || ($_[5] == 0)) { |
34 | # we do not touch roots files or directories. |
35 | return; |
36 | } |
37 | if ($_[2] > (time() - ($minage * 3600))) { |
38 | #print("not adding $_[0], too new\n"); |
39 | return; |
40 | } |
41 | $totalfilesseen++; |
42 | push(@allentries, [ $_[0], $_[2], $_[3], $_[4], $_[5], $_[6] ]); |
43 | } |
44 | |
45 | # Par. 0: Path were we start |
46 | # Par. 1: Recursion counter |
47 | # Returns: Number of files/dirs found (non-recursively!) |
48 | sub runrecursive($$); |
49 | sub runrecursive($$) { |
50 | my $DIR; |
51 | my $pth = $_[0]; |
52 | my $curdirentry; |
53 | my @statres; |
54 | my $recctr = $_[1]; |
55 | my $direntries = 0; |
56 | unless (opendir($DIR, $pth)) { |
57 | print("ERROR: Failed to open dir $pth\n"); |
58 | exit(1); |
59 | } |
60 | while ($curdirentrys = readdir($DIR)) { |
61 | if (($curdirentrys eq '.') || ($curdirentrys eq '..')) { |
62 | next; |
63 | } |
64 | $direntries++; |
65 | $curdirentryf = $pth.'/'.$curdirentrys; |
66 | @statres = lstat($curdirentryf); |
67 | unless (@statres > 12) { |
68 | # This usually happens if we have a symlink to a file for which we have |
69 | # no permission to read. |
70 | #print("stat failed for $curdirentry\n"); |
71 | next; |
72 | } |
73 | if (S_ISLNK($statres[2])) { next; } # A symlink? We no like. |
74 | if ($recctr <= 4) { # There should be no files on these levels |
75 | unless (S_ISDIR($statres[2])) { next; } |
76 | unless ($curdirentrys =~ m/^\d+$/) { next; } |
77 | } else { # whereas there should be only files called .meta on level 5. |
78 | unless (S_ISREG($statres[2])) { next; } |
79 | unless ($curdirentrys =~ m/^\d+\.meta$/) { next; } |
80 | } |
81 | if ($recctr == 0) { |
82 | my $z = int($curdirentrys); |
83 | unless (($z >= $zoommin) && ($z <= $zoommax)) { next; } |
84 | runrecursive($curdirentryf, $recctr+1); |
85 | } elsif ($recctr <= 4) { |
86 | if (runrecursive($curdirentryf, $recctr+1) == 0) { # Empty dir! |
87 | if ($rmemptydirs) { |
88 | print("RMDIR: $curdirentryf\n"); |
89 | rmdir($curdirentryf); |
90 | } |
91 | } |
92 | } else { |
93 | remembercandidate($curdirentryf, $statres[8], $statres[9], $statres[2], $statres[4], $statres[5], $statres[12]); |
94 | } |
95 | } |
96 | closedir($DIR); |
97 | return $direntries; |
98 | } |
99 | |
100 | # Par. 0: Full path of file |
101 | sub rerenderfile($) { |
102 | my $fpth = $_[0]; |
103 | # We first need to figure out the relevant numbers from the full file path |
104 | unless ($fpth =~ m!/(\d+)/(\d+)/(\d+)/(\d+)/(\d+)/(\d+)\.meta$!) { |
105 | print("WARNING: rerenderfile: Failed to extract path components for rerendering out of '$fpth' - skipping\n"); |
106 | return; |
107 | } |
108 | my $zl = $1; my @p = ( $2, $3, $4, $5, $6 ); |
109 | my $calcx = 0; my $calcy = 0; |
110 | my $i; |
111 | for ($i = 0; $i < 5; $i++) { |
112 | $calcx = ($calcx << 4) | (($p[$i] & 0xf0) >> 4); |
113 | $calcy = ($calcy << 4) | (($p[$i] & 0x0f) >> 0); |
114 | } |
115 | # struct meta_layout { |
116 | # char magic[4]; // 'M' 'E' 'T' 'A' |
117 | # int count; // METATILE ^ 2 |
118 | # int x, y, z; // lowest x,y of this metatile, plus z |
119 | my $fi; |
120 | unless (open($fi, '<', $fpth)) { |
121 | print("WARNING: rerenderfile: Failed to open file '$fpth' - skipping\n"); |
122 | return; |
123 | } |
124 | binmode($fi); |
125 | my $dbuf; |
126 | unless (read($fi, $dbuf, 20)) { |
127 | print("WARNING: rerenderfile: Failed to read info from metatile '$fpth' - skipping\n"); |
128 | return; |
129 | } |
130 | close($fi); undef($fi); |
131 | unless (substr($dbuf, 0, 4) eq 'META') { |
132 | print("WARNING: rerenderfile: file '$fpth' is not a metatile - skipping\n"); |
133 | return; |
134 | } |
135 | unless (ord(substr($dbuf, 4, 1)) == 64) { |
136 | print("WARNING: rerenderfile: file '$fpth' is not a 8x8 metatile - skipping\n"); |
137 | return; |
138 | } |
139 | my $fx = (ord(substr($dbuf, 8, 1)) << 0) | (ord(substr($dbuf, 9, 1)) << 8) |
140 | | (ord(substr($dbuf, 10, 1)) << 16) | (ord(substr($dbuf, 11, 1)) << 24); |
141 | unless ($fx == $calcx) { |
142 | print("WARNING: rerenderfile: file '$fpth' is invalid - xsize $fx != $calcx - skipping\n"); |
143 | return; |
144 | } |
145 | my $fy = (ord(substr($dbuf, 12, 1)) << 0) | (ord(substr($dbuf, 13, 1)) << 8) |
146 | | (ord(substr($dbuf, 14, 1)) << 16) | (ord(substr($dbuf, 15, 1)) << 24); |
147 | unless ($fy == $calcy) { |
148 | print("WARNING: rerenderfile: file '$fpth' is invalid - xsize $fy != $calcy - skipping\n"); |
149 | return; |
150 | } |
151 | my $fz = (ord(substr($dbuf, 16, 1)) << 0) | (ord(substr($dbuf, 17, 1)) << 8) |
152 | | (ord(substr($dbuf, 18, 1)) << 16) | (ord(substr($dbuf, 19, 1)) << 24); |
153 | unless ($fz == $zl) { |
154 | print("WARNING: rerenderfile: file '$fpth' is invalid - zsize $fz != $zl - skipping\n"); |
155 | return; |
156 | } |
157 | print("Sending rendering request for z=$zl x=$calcx y=$calcy to regenerate '$fpth'\n"); |
158 | if (system("$rrs $zl $calcx $calcy")) { |
159 | print("Error executing $rrs $zl $calcx $calcy\n"); |
160 | } |
161 | } |
162 | |
163 | # Par. 0: x |
164 | # Par. 1: y |
165 | sub calcpathfromcomponents($$) { |
166 | my @res = (); |
167 | my $i; my $x = $_[0]; my $y = $_[1]; |
168 | for ($i = 4; $i >= 0; $i--) { |
169 | $res[$i] = sprintf("%d", (($x & 0x0f) << 4) + ($y & 0x0f)); |
170 | $x = $x >> 4; |
171 | $y = $y >> 4; |
172 | } |
173 | return $res[0] . "/" . $res[1] . "/" . $res[2] . "/" . $res[3] . "/" . $res[4]; |
174 | } |
175 | |
176 | sub dorerenderexpiredlist() { |
177 | my $ll; |
178 | %rerenderlist = (); |
179 | while ($ll = <STDIN>) { |
180 | if ($ll =~ m!^(\d+)/(\d+)/(\d+)$!) { |
181 | my $x = $2; my $y = $3; my $z = $1; |
182 | #print("Handling z=$z x=$x y=$y\n"); |
183 | if ($z != ($zoommax - 3)) { |
184 | print("Ignoring z=$z x=$x y=$y because of wrong zoom\n"); |
185 | next; |
186 | } |
187 | my $cz; my $cx; my $cy; my $curz; |
188 | for ($curz = $zoommin; $curz <= $zoommax; $curz++) { |
189 | $cz = $z; $cx = $x; $cy = $y; |
190 | while ($cz < $curz) { |
191 | $cz++; $cx <<= 1; $cy <<= 1; |
192 | } |
193 | while ($cz > $curz) { |
194 | $cz--; $cx >>= 1; $cy >>= 1; |
195 | } |
196 | #print("Matching tile at z=$cz: x=$cx y=$cy -"); |
197 | $cx = $cx & 0xfff8; $cy = $cy & 0xfff8; |
198 | #print(" rounded to x=$cx y=$cy\n"); |
199 | $rerenderlist{$cz}{$cx}{$cy} = 1; |
200 | } |
201 | } |
202 | } |
203 | $filesdone = 0; $filesseen = 0; |
204 | foreach $z (sort(keys(%rerenderlist))) { |
205 | foreach $x (sort(keys(%{$rerenderlist{$z}}))) { |
206 | foreach $y (sort(keys(%{$rerenderlist{$z}{$x}}))) { |
207 | my $p = ${fspath} . '/' . $z . '/' . calcpathfromcomponents($x, $y) . '.meta'; |
208 | #print("Checking: $z $x $y - Path: $p\n"); |
209 | $filesseen++; |
210 | if (-e $p) { |
211 | $filesdone++; |
212 | print("Sending rendering request for z=$z x=$x y=$y to regenerate '$p'\n"); |
213 | if (system("$rrs $z $x $y")) { |
214 | print("Error executing $rrs $z $x $y\n"); |
215 | } |
216 | } |
217 | } |
218 | } |
219 | } |
220 | print("Sent re-rendering-requests for $filesdone files that actually existed (of $filesseen candidates)\n"); |
221 | } |
222 | |
223 | $fspath = ''; |
224 | for ($i = 0; $i < @ARGV; $i++) { |
225 | $curarg = $ARGV[$i]; |
226 | if ($curarg eq '--help') { |
227 | print("Syntax: $0 [--help] [--zoom z] [--limit n] [--action a] [--rmemptydirs] [--rrs path] [--minage hrs] directory\n"); |
228 | print(" --zoom z[-z2]: Zoom level(s) to handle (default: 0-20).\n"); |
229 | print(" --limit n: Limit to the n oldest files (default: no limit)\n"); |
230 | print(" --action what: what action to perform with the found files.\n"); |
231 | print(" valid actions are: print delete rerender rerenderexpiredlist (default: print)\n"); |
232 | print(" rerenderexpiredlist: this is very different from all other commands, as it\n"); |
233 | print(" expects to read a list of expired tiles to rerender on STDIN. The list has\n"); |
234 | print(" to be in the format that osm2pgsql spits out.\n"); |
235 | print(" --rmemptydirs: remove empty subdirectories\n"); |
236 | print(" --rrs path: if action is rerender, this gives the path to the rerender script.\n"); |
237 | print(" it gets called with three parameters: z x y\n"); |
238 | print(" --minage hrs: only care about files that are at least hrs hours old\n"); |
239 | exit(1); |
240 | } elsif ($curarg eq '--zoom') { |
241 | $i++; |
242 | if ($i >= @ARGV) { |
243 | print("Error: --zoom requires a parameter\n"); |
244 | exit(1); |
245 | } else { |
246 | if ($ARGV[$i] =~ m/^(\d+)-(\d+)$/) { |
247 | $zoommin = $1; $zoommax = $2; |
248 | } elsif ($ARGV[$i] =~ m/^(\d+)$/) { |
249 | $zoommin = $1; $zoommax = $1; |
250 | } else { |
251 | print("Error: --zoom needs a single numeric parameter or a range (e.g. 4 or 10-12)\n"); |
252 | exit(1); |
253 | } |
254 | } |
255 | } elsif ($curarg eq '--limit') { |
256 | $i++; |
257 | if ($i >= @ARGV) { |
258 | print("Error: --limit requires a parameter\n"); |
259 | exit(1); |
260 | } else { |
261 | if ($ARGV[$i] =~ m/^(\d+)$/) { |
262 | $filelimit = $1; |
263 | } else { |
264 | print("Error: --limit requires an positive integer as parameter.\n"); |
265 | } |
266 | } |
267 | } elsif ($curarg eq '--action') { |
268 | $i++; |
269 | if ($i >= @ARGV) { |
270 | print("Error: --action requires a parameter\n"); |
271 | exit(1); |
272 | } else { |
273 | if ($ARGV[$i] eq 'print') { |
274 | $action = 0; |
275 | } elsif ($ARGV[$i] eq 'delete') { |
276 | $action = 1; |
277 | } elsif ($ARGV[$i] eq 'rerender') { |
278 | $action = 2; |
279 | } elsif ($ARGV[$i] eq 'rerenderexpiredlist') { |
280 | $action = 3; |
281 | } else { |
282 | print("Error: Invalid action selected.\n"); |
283 | exit(1); |
284 | } |
285 | } |
286 | } elsif ($curarg eq '--rrs') { |
287 | $i++; |
288 | if ($i >= @ARGV) { |
289 | print("Error: --rrs requires a parameter\n"); |
290 | exit(1); |
291 | } else { |
292 | $rrs = $ARGV[$i]; |
293 | } |
294 | } elsif ($curarg eq '--minage') { |
295 | $i++; |
296 | if ($i >= @ARGV) { |
297 | print("Error: --minage requires a parameter\n"); |
298 | exit(1); |
299 | } else { |
300 | if ($ARGV[$i] =~ m/^(\d+)$/) { |
301 | $minage = $1; |
302 | } else { |
303 | print("Error: --minage requires an positive integer as parameter.\n"); |
304 | } |
305 | } |
306 | } elsif ($curarg eq '--rmemptydirs') { |
307 | $rmemptydirs = 1; |
308 | } else { |
309 | if ($fspath eq '') { |
310 | $fspath = $curarg; |
311 | } else { |
312 | print("Too many parameters, or parameter(s) not understood.\n"); |
313 | print("I can only handle one directory parameter, but I consider both "); |
314 | print("'$curarg' and '$fspath' a pathname since they are not a known parameter.\n"); |
315 | exit(1); |
316 | } |
317 | } |
318 | } |
319 | if ($fspath eq '') { |
320 | print("ERROR: No path to clear given.\n"); |
321 | exit(1); |
322 | } |
323 | if ($action == 3) { # This significantly differs from the rest of our operations |
324 | dorerenderexpiredlist(); |
325 | exit(0); |
326 | } |
327 | @allentries = (); |
328 | $totalfilesseen = 0; |
329 | runrecursive($fspath, 0); |
330 | @allentries = sort(sortbydateasc @allentries); |
331 | print(int(@allentries)." files seen\n"); |
332 | $filesdone = 0; |
333 | foreach $ent (@allentries) { |
334 | if ($filelimit > 0) { |
335 | if ($filesdone >= $filelimit) { |
336 | print("File limit $filelimit reached, exiting.\n"); |
337 | exit(0); |
338 | } |
339 | } |
340 | #print("Handling File '$ent->[0]', $ent->[5] blocks [u: $ent->[3] g: $ent->[4] mtime: ".strftime("%Y-%m-%d.%H:%M:%S", localtime($ent->[1]))."]\n"); |
341 | if ($action == 0) { # Print |
342 | print("$ent->[0]\n"); |
343 | } elsif ($action == 1) { # Delete |
344 | print("DELETING $ent->[0] (mtime " . strftime("%Y-%m-%d.%H:%M:%S", localtime($ent->[1])) . ")\n"); |
345 | unless (unlink($ent->[0])) { |
346 | print("ERROR: rm for $ent->[0] failed!\n"); |
347 | } |
348 | } elsif ($action == 2) { # Rerender |
349 | rerenderfile($ent->[0]); |
350 | } |
351 | $filesdone++; |
352 | } |