]>
Commit | Line | Data |
---|---|---|
1 | #!/usr/bin/perl -w | |
2 | ||
3 | # OSM tile cleanup / maintenance script. Big parts are a copied from | |
4 | # hochwasserloeschung.pl. | |
5 | ||
6 | use Fcntl ':mode'; | |
7 | use POSIX 'strftime'; | |
8 | ||
9 | $zoommin = 0; | |
10 | $zoommax = 20; | |
11 | $filelimit = -1; | |
12 | $rmemptydirs = 0; | |
13 | $action = 0; | |
14 | $rrs = '/bin/echo'; | |
15 | $minage = 0; # In hours! | |
16 | ||
17 | # Sort function for candidate list | |
18 | sub sortbydateasc { | |
19 | #print("$a->[1] $b->[1]\n"); | |
20 | return ($a->[1] <=> $b->[1]); | |
21 | } | |
22 | ||
23 | # Remembers a candidate for deletion, if required sorting and purging out the | |
24 | # list. | |
25 | # Parameters: 0 filename | |
26 | # 1 atime | |
27 | # 2 mtime | |
28 | # 3 type field from stat (containing DIR, LINK etc.) | |
29 | # 4 uid | |
30 | # 5 gid | |
31 | # 6 size in bytes | |
32 | sub remembercandidate($$$$$$$) { | |
33 | if (($_[4] == 0) || ($_[5] == 0)) { | |
34 | # we do not touch roots files or directories. | |
35 | return; | |
36 | } | |
37 | if ($_[2] > (time() - ($minage * 3600))) { | |
38 | #print("not adding $_[0], too new\n"); | |
39 | return; | |
40 | } | |
41 | $totalfilesseen++; | |
42 | push(@allentries, [ $_[0], $_[2], $_[3], $_[4], $_[5], $_[6] ]); | |
43 | } | |
44 | ||
45 | # Par. 0: Path were we start | |
46 | # Par. 1: Recursion counter | |
47 | # Returns: Number of files/dirs found (non-recursively!) | |
48 | sub runrecursive($$); | |
49 | sub runrecursive($$) { | |
50 | my $DIR; | |
51 | my $pth = $_[0]; | |
52 | my $curdirentry; | |
53 | my @statres; | |
54 | my $recctr = $_[1]; | |
55 | my $direntries = 0; | |
56 | unless (opendir($DIR, $pth)) { | |
57 | print("ERROR: Failed to open dir $pth\n"); | |
58 | exit(1); | |
59 | } | |
60 | while ($curdirentrys = readdir($DIR)) { | |
61 | if (($curdirentrys eq '.') || ($curdirentrys eq '..')) { | |
62 | next; | |
63 | } | |
64 | $direntries++; | |
65 | $curdirentryf = $pth.'/'.$curdirentrys; | |
66 | @statres = lstat($curdirentryf); | |
67 | unless (@statres > 12) { | |
68 | # This usually happens if we have a symlink to a file for which we have | |
69 | # no permission to read. | |
70 | #print("stat failed for $curdirentry\n"); | |
71 | next; | |
72 | } | |
73 | if (S_ISLNK($statres[2])) { next; } # A symlink? We no like. | |
74 | if ($recctr <= 4) { # There should be no files on these levels | |
75 | unless (S_ISDIR($statres[2])) { next; } | |
76 | unless ($curdirentrys =~ m/^\d+$/) { next; } | |
77 | } else { # whereas there should be only files called .meta on level 5. | |
78 | unless (S_ISREG($statres[2])) { next; } | |
79 | unless ($curdirentrys =~ m/^\d+\.meta$/) { next; } | |
80 | } | |
81 | if ($recctr == 0) { | |
82 | my $z = int($curdirentrys); | |
83 | unless (($z >= $zoommin) && ($z <= $zoommax)) { next; } | |
84 | runrecursive($curdirentryf, $recctr+1); | |
85 | } elsif ($recctr <= 4) { | |
86 | if (runrecursive($curdirentryf, $recctr+1) == 0) { # Empty dir! | |
87 | if ($rmemptydirs) { | |
88 | print("RMDIR: $curdirentryf\n"); | |
89 | rmdir($curdirentryf); | |
90 | } | |
91 | } | |
92 | } else { | |
93 | remembercandidate($curdirentryf, $statres[8], $statres[9], $statres[2], $statres[4], $statres[5], $statres[12]); | |
94 | } | |
95 | } | |
96 | closedir($DIR); | |
97 | return $direntries; | |
98 | } | |
99 | ||
100 | # Par. 0: Full path of file | |
101 | sub rerenderfile($) { | |
102 | my $fpth = $_[0]; | |
103 | # We first need to figure out the relevant numbers from the full file path | |
104 | unless ($fpth =~ m!/(\d+)/(\d+)/(\d+)/(\d+)/(\d+)/(\d+)\.meta$!) { | |
105 | print("WARNING: rerenderfile: Failed to extract path components for rerendering out of '$fpth' - skipping\n"); | |
106 | return; | |
107 | } | |
108 | my $zl = $1; my @p = ( $2, $3, $4, $5, $6 ); | |
109 | my $calcx = 0; my $calcy = 0; | |
110 | my $i; | |
111 | for ($i = 0; $i < 5; $i++) { | |
112 | $calcx = ($calcx << 4) | (($p[$i] & 0xf0) >> 4); | |
113 | $calcy = ($calcy << 4) | (($p[$i] & 0x0f) >> 0); | |
114 | } | |
115 | # struct meta_layout { | |
116 | # char magic[4]; // 'M' 'E' 'T' 'A' | |
117 | # int count; // METATILE ^ 2 | |
118 | # int x, y, z; // lowest x,y of this metatile, plus z | |
119 | my $fi; | |
120 | unless (open($fi, '<', $fpth)) { | |
121 | print("WARNING: rerenderfile: Failed to open file '$fpth' - skipping\n"); | |
122 | return; | |
123 | } | |
124 | binmode($fi); | |
125 | my $dbuf; | |
126 | unless (read($fi, $dbuf, 20)) { | |
127 | print("WARNING: rerenderfile: Failed to read info from metatile '$fpth' - skipping\n"); | |
128 | return; | |
129 | } | |
130 | close($fi); undef($fi); | |
131 | unless (substr($dbuf, 0, 4) eq 'META') { | |
132 | print("WARNING: rerenderfile: file '$fpth' is not a metatile - skipping\n"); | |
133 | return; | |
134 | } | |
135 | unless (ord(substr($dbuf, 4, 1)) == 64) { | |
136 | print("WARNING: rerenderfile: file '$fpth' is not a 8x8 metatile - skipping\n"); | |
137 | return; | |
138 | } | |
139 | my $fx = (ord(substr($dbuf, 8, 1)) << 0) | (ord(substr($dbuf, 9, 1)) << 8) | |
140 | | (ord(substr($dbuf, 10, 1)) << 16) | (ord(substr($dbuf, 11, 1)) << 24); | |
141 | unless ($fx == $calcx) { | |
142 | print("WARNING: rerenderfile: file '$fpth' is invalid - xsize $fx != $calcx - skipping\n"); | |
143 | return; | |
144 | } | |
145 | my $fy = (ord(substr($dbuf, 12, 1)) << 0) | (ord(substr($dbuf, 13, 1)) << 8) | |
146 | | (ord(substr($dbuf, 14, 1)) << 16) | (ord(substr($dbuf, 15, 1)) << 24); | |
147 | unless ($fy == $calcy) { | |
148 | print("WARNING: rerenderfile: file '$fpth' is invalid - xsize $fy != $calcy - skipping\n"); | |
149 | return; | |
150 | } | |
151 | my $fz = (ord(substr($dbuf, 16, 1)) << 0) | (ord(substr($dbuf, 17, 1)) << 8) | |
152 | | (ord(substr($dbuf, 18, 1)) << 16) | (ord(substr($dbuf, 19, 1)) << 24); | |
153 | unless ($fz == $zl) { | |
154 | print("WARNING: rerenderfile: file '$fpth' is invalid - zsize $fz != $zl - skipping\n"); | |
155 | return; | |
156 | } | |
157 | print("Sending rendering request for z=$zl x=$calcx y=$calcy to regenerate '$fpth'\n"); | |
158 | if (system("$rrs $zl $calcx $calcy")) { | |
159 | print("Error executing $rrs $zl $calcx $calcy\n"); | |
160 | } | |
161 | } | |
162 | ||
163 | # Par. 0: x | |
164 | # Par. 1: y | |
165 | sub calcpathfromcomponents($$) { | |
166 | my @res = (); | |
167 | my $i; my $x = $_[0]; my $y = $_[1]; | |
168 | for ($i = 4; $i >= 0; $i--) { | |
169 | $res[$i] = sprintf("%d", (($x & 0x0f) << 4) + ($y & 0x0f)); | |
170 | $x = $x >> 4; | |
171 | $y = $y >> 4; | |
172 | } | |
173 | return $res[0] . "/" . $res[1] . "/" . $res[2] . "/" . $res[3] . "/" . $res[4]; | |
174 | } | |
175 | ||
176 | sub dorerenderexpiredlist() { | |
177 | my $ll; | |
178 | %rerenderlist = (); | |
179 | while ($ll = <STDIN>) { | |
180 | if ($ll =~ m!^(\d+)/(\d+)/(\d+)$!) { | |
181 | my $x = $2; my $y = $3; my $z = $1; | |
182 | #print("Handling z=$z x=$x y=$y\n"); | |
183 | if ($z != ($zoommax - 3)) { | |
184 | print("Ignoring z=$z x=$x y=$y because of wrong zoom\n"); | |
185 | next; | |
186 | } | |
187 | my $cz; my $cx; my $cy; my $curz; | |
188 | for ($curz = $zoommin; $curz <= $zoommax; $curz++) { | |
189 | $cz = $z; $cx = $x; $cy = $y; | |
190 | while ($cz < $curz) { | |
191 | $cz++; $cx <<= 1; $cy <<= 1; | |
192 | } | |
193 | while ($cz > $curz) { | |
194 | $cz--; $cx >>= 1; $cy >>= 1; | |
195 | } | |
196 | #print("Matching tile at z=$cz: x=$cx y=$cy -"); | |
197 | $cx = $cx & 0xfff8; $cy = $cy & 0xfff8; | |
198 | #print(" rounded to x=$cx y=$cy\n"); | |
199 | $rerenderlist{$cz}{$cx}{$cy} = 1; | |
200 | } | |
201 | } | |
202 | } | |
203 | $filesdone = 0; $filesseen = 0; | |
204 | foreach $z (sort(keys(%rerenderlist))) { | |
205 | foreach $x (sort(keys(%{$rerenderlist{$z}}))) { | |
206 | foreach $y (sort(keys(%{$rerenderlist{$z}{$x}}))) { | |
207 | my $p = ${fspath} . '/' . $z . '/' . calcpathfromcomponents($x, $y) . '.meta'; | |
208 | #print("Checking: $z $x $y - Path: $p\n"); | |
209 | $filesseen++; | |
210 | if (-e $p) { | |
211 | $filesdone++; | |
212 | print("Sending rendering request for z=$z x=$x y=$y to regenerate '$p'\n"); | |
213 | if (system("$rrs $z $x $y")) { | |
214 | print("Error executing $rrs $z $x $y\n"); | |
215 | } | |
216 | } | |
217 | } | |
218 | } | |
219 | } | |
220 | print("Sent re-rendering-requests for $filesdone files that actually existed (of $filesseen candidates)\n"); | |
221 | } | |
222 | ||
223 | $fspath = ''; | |
224 | for ($i = 0; $i < @ARGV; $i++) { | |
225 | $curarg = $ARGV[$i]; | |
226 | if ($curarg eq '--help') { | |
227 | print("Syntax: $0 [--help] [--zoom z] [--limit n] [--action a] [--rmemptydirs] [--rrs path] [--minage hrs] directory\n"); | |
228 | print(" --zoom z[-z2]: Zoom level(s) to handle (default: 0-20).\n"); | |
229 | print(" --limit n: Limit to the n oldest files (default: no limit)\n"); | |
230 | print(" --action what: what action to perform with the found files.\n"); | |
231 | print(" valid actions are: print delete rerender rerenderexpiredlist (default: print)\n"); | |
232 | print(" rerenderexpiredlist: this is very different from all other commands, as it\n"); | |
233 | print(" expects to read a list of expired tiles to rerender on STDIN. The list has\n"); | |
234 | print(" to be in the format that osm2pgsql spits out.\n"); | |
235 | print(" --rmemptydirs: remove empty subdirectories\n"); | |
236 | print(" --rrs path: if action is rerender, this gives the path to the rerender script.\n"); | |
237 | print(" it gets called with three parameters: z x y\n"); | |
238 | print(" --minage hrs: only care about files that are at least hrs hours old\n"); | |
239 | exit(1); | |
240 | } elsif ($curarg eq '--zoom') { | |
241 | $i++; | |
242 | if ($i >= @ARGV) { | |
243 | print("Error: --zoom requires a parameter\n"); | |
244 | exit(1); | |
245 | } else { | |
246 | if ($ARGV[$i] =~ m/^(\d+)-(\d+)$/) { | |
247 | $zoommin = $1; $zoommax = $2; | |
248 | } elsif ($ARGV[$i] =~ m/^(\d+)$/) { | |
249 | $zoommin = $1; $zoommax = $1; | |
250 | } else { | |
251 | print("Error: --zoom needs a single numeric parameter or a range (e.g. 4 or 10-12)\n"); | |
252 | exit(1); | |
253 | } | |
254 | } | |
255 | } elsif ($curarg eq '--limit') { | |
256 | $i++; | |
257 | if ($i >= @ARGV) { | |
258 | print("Error: --limit requires a parameter\n"); | |
259 | exit(1); | |
260 | } else { | |
261 | if ($ARGV[$i] =~ m/^(\d+)$/) { | |
262 | $filelimit = $1; | |
263 | } else { | |
264 | print("Error: --limit requires an positive integer as parameter.\n"); | |
265 | } | |
266 | } | |
267 | } elsif ($curarg eq '--action') { | |
268 | $i++; | |
269 | if ($i >= @ARGV) { | |
270 | print("Error: --action requires a parameter\n"); | |
271 | exit(1); | |
272 | } else { | |
273 | if ($ARGV[$i] eq 'print') { | |
274 | $action = 0; | |
275 | } elsif ($ARGV[$i] eq 'delete') { | |
276 | $action = 1; | |
277 | } elsif ($ARGV[$i] eq 'rerender') { | |
278 | $action = 2; | |
279 | } elsif ($ARGV[$i] eq 'rerenderexpiredlist') { | |
280 | $action = 3; | |
281 | } else { | |
282 | print("Error: Invalid action selected.\n"); | |
283 | exit(1); | |
284 | } | |
285 | } | |
286 | } elsif ($curarg eq '--rrs') { | |
287 | $i++; | |
288 | if ($i >= @ARGV) { | |
289 | print("Error: --rrs requires a parameter\n"); | |
290 | exit(1); | |
291 | } else { | |
292 | $rrs = $ARGV[$i]; | |
293 | } | |
294 | } elsif ($curarg eq '--minage') { | |
295 | $i++; | |
296 | if ($i >= @ARGV) { | |
297 | print("Error: --minage requires a parameter\n"); | |
298 | exit(1); | |
299 | } else { | |
300 | if ($ARGV[$i] =~ m/^(\d+)$/) { | |
301 | $minage = $1; | |
302 | } else { | |
303 | print("Error: --minage requires an positive integer as parameter.\n"); | |
304 | } | |
305 | } | |
306 | } elsif ($curarg eq '--rmemptydirs') { | |
307 | $rmemptydirs = 1; | |
308 | } else { | |
309 | if ($fspath eq '') { | |
310 | $fspath = $curarg; | |
311 | } else { | |
312 | print("Too many parameters, or parameter(s) not understood.\n"); | |
313 | print("I can only handle one directory parameter, but I consider both "); | |
314 | print("'$curarg' and '$fspath' a pathname since they are not a known parameter.\n"); | |
315 | exit(1); | |
316 | } | |
317 | } | |
318 | } | |
319 | if ($fspath eq '') { | |
320 | print("ERROR: No path to clear given.\n"); | |
321 | exit(1); | |
322 | } | |
323 | if ($action == 3) { # This significantly differs from the rest of our operations | |
324 | dorerenderexpiredlist(); | |
325 | exit(0); | |
326 | } | |
327 | @allentries = (); | |
328 | $totalfilesseen = 0; | |
329 | runrecursive($fspath, 0); | |
330 | @allentries = sort(sortbydateasc @allentries); | |
331 | print(int(@allentries)." files seen\n"); | |
332 | $filesdone = 0; | |
333 | foreach $ent (@allentries) { | |
334 | if ($filelimit > 0) { | |
335 | if ($filesdone >= $filelimit) { | |
336 | print("File limit $filelimit reached, exiting.\n"); | |
337 | exit(0); | |
338 | } | |
339 | } | |
340 | #print("Handling File '$ent->[0]', $ent->[5] blocks [u: $ent->[3] g: $ent->[4] mtime: ".strftime("%Y-%m-%d.%H:%M:%S", localtime($ent->[1]))."]\n"); | |
341 | if ($action == 0) { # Print | |
342 | print("$ent->[0]\n"); | |
343 | } elsif ($action == 1) { # Delete | |
344 | print("DELETING $ent->[0] (mtime " . strftime("%Y-%m-%d.%H:%M:%S", localtime($ent->[1])) . ")\n"); | |
345 | unless (unlink($ent->[0])) { | |
346 | print("ERROR: rm for $ent->[0] failed!\n"); | |
347 | } | |
348 | } elsif ($action == 2) { # Rerender | |
349 | rerenderfile($ent->[0]); | |
350 | } | |
351 | $filesdone++; | |
352 | } |