Merge branch 'master' of git.rrze.uni-erlangen.de:osmrrze
[osmrrze.git] / scripts / osmtilecleanup.pl
... / ...
CommitLineData
1#!/usr/bin/perl -w
2
3# OSM tile cleanup / maintenance script. Big parts are a copied from
4# hochwasserloeschung.pl.
5
6use Fcntl ':mode';
7use POSIX 'strftime';
8
9$zoommin = 0;
10$zoommax = 20;
11$filelimit = -1;
12$rmemptydirs = 0;
13$action = 0;
14$rrs = '/bin/echo';
15$minage = 0; # In hours!
16
17# Sort function for candidate list
18sub sortbydateasc {
19 #print("$a->[1] $b->[1]\n");
20 return ($a->[1] <=> $b->[1]);
21}
22
23# Remembers a candidate for deletion, if required sorting and purging out the
24# list.
25# Parameters: 0 filename
26# 1 atime
27# 2 mtime
28# 3 type field from stat (containing DIR, LINK etc.)
29# 4 uid
30# 5 gid
31# 6 size in bytes
32sub remembercandidate($$$$$$$) {
33 if (($_[4] == 0) || ($_[5] == 0)) {
34 # we do not touch roots files or directories.
35 return;
36 }
37 if ($_[2] > (time() - ($minage * 3600))) {
38 #print("not adding $_[0], too new\n");
39 return;
40 }
41 $totalfilesseen++;
42 push(@allentries, [ $_[0], $_[2], $_[3], $_[4], $_[5], $_[6] ]);
43}
44
45# Par. 0: Path were we start
46# Par. 1: Recursion counter
47# Returns: Number of files/dirs found (non-recursively!)
48sub runrecursive($$);
49sub runrecursive($$) {
50 my $DIR;
51 my $pth = $_[0];
52 my $curdirentry;
53 my @statres;
54 my $recctr = $_[1];
55 my $direntries = 0;
56 unless (opendir($DIR, $pth)) {
57 print("ERROR: Failed to open dir $pth\n");
58 exit(1);
59 }
60 while ($curdirentrys = readdir($DIR)) {
61 if (($curdirentrys eq '.') || ($curdirentrys eq '..')) {
62 next;
63 }
64 $direntries++;
65 $curdirentryf = $pth.'/'.$curdirentrys;
66 @statres = lstat($curdirentryf);
67 unless (@statres > 12) {
68 # This usually happens if we have a symlink to a file for which we have
69 # no permission to read.
70 #print("stat failed for $curdirentry\n");
71 next;
72 }
73 if (S_ISLNK($statres[2])) { next; } # A symlink? We no like.
74 if ($recctr <= 4) { # There should be no files on these levels
75 unless (S_ISDIR($statres[2])) { next; }
76 unless ($curdirentrys =~ m/^\d+$/) { next; }
77 } else { # whereas there should be only files called .meta on level 5.
78 unless (S_ISREG($statres[2])) { next; }
79 unless ($curdirentrys =~ m/^\d+\.meta$/) { next; }
80 }
81 if ($recctr == 0) {
82 my $z = int($curdirentrys);
83 unless (($z >= $zoommin) && ($z <= $zoommax)) { next; }
84 runrecursive($curdirentryf, $recctr+1);
85 } elsif ($recctr <= 4) {
86 if (runrecursive($curdirentryf, $recctr+1) == 0) { # Empty dir!
87 if ($rmemptydirs) {
88 print("RMDIR: $curdirentryf\n");
89 rmdir($curdirentryf);
90 }
91 }
92 } else {
93 remembercandidate($curdirentryf, $statres[8], $statres[9], $statres[2], $statres[4], $statres[5], $statres[12]);
94 }
95 }
96 closedir($DIR);
97 return $direntries;
98}
99
100# Par. 0: Full path of file
101sub rerenderfile($) {
102 my $fpth = $_[0];
103 # We first need to figure out the relevant numbers from the full file path
104 unless ($fpth =~ m!/(\d+)/(\d+)/(\d+)/(\d+)/(\d+)/(\d+)\.meta$!) {
105 print("WARNING: rerenderfile: Failed to extract path components for rerendering out of '$fpth' - skipping\n");
106 return;
107 }
108 my $zl = $1; my @p = ( $2, $3, $4, $5, $6 );
109 my $calcx = 0; my $calcy = 0;
110 my $i;
111 for ($i = 0; $i < 5; $i++) {
112 $calcx = ($calcx << 4) | (($p[$i] & 0xf0) >> 4);
113 $calcy = ($calcy << 4) | (($p[$i] & 0x0f) >> 0);
114 }
115 # struct meta_layout {
116 # char magic[4]; // 'M' 'E' 'T' 'A'
117 # int count; // METATILE ^ 2
118 # int x, y, z; // lowest x,y of this metatile, plus z
119 my $fi;
120 unless (open($fi, '<', $fpth)) {
121 print("WARNING: rerenderfile: Failed to open file '$fpth' - skipping\n");
122 return;
123 }
124 binmode($fi);
125 my $dbuf;
126 unless (read($fi, $dbuf, 20)) {
127 print("WARNING: rerenderfile: Failed to read info from metatile '$fpth' - skipping\n");
128 return;
129 }
130 close($fi); undef($fi);
131 unless (substr($dbuf, 0, 4) eq 'META') {
132 print("WARNING: rerenderfile: file '$fpth' is not a metatile - skipping\n");
133 return;
134 }
135 unless (ord(substr($dbuf, 4, 1)) == 64) {
136 print("WARNING: rerenderfile: file '$fpth' is not a 8x8 metatile - skipping\n");
137 return;
138 }
139 my $fx = (ord(substr($dbuf, 8, 1)) << 0) | (ord(substr($dbuf, 9, 1)) << 8)
140 | (ord(substr($dbuf, 10, 1)) << 16) | (ord(substr($dbuf, 11, 1)) << 24);
141 unless ($fx == $calcx) {
142 print("WARNING: rerenderfile: file '$fpth' is invalid - xsize $fx != $calcx - skipping\n");
143 return;
144 }
145 my $fy = (ord(substr($dbuf, 12, 1)) << 0) | (ord(substr($dbuf, 13, 1)) << 8)
146 | (ord(substr($dbuf, 14, 1)) << 16) | (ord(substr($dbuf, 15, 1)) << 24);
147 unless ($fy == $calcy) {
148 print("WARNING: rerenderfile: file '$fpth' is invalid - xsize $fy != $calcy - skipping\n");
149 return;
150 }
151 my $fz = (ord(substr($dbuf, 16, 1)) << 0) | (ord(substr($dbuf, 17, 1)) << 8)
152 | (ord(substr($dbuf, 18, 1)) << 16) | (ord(substr($dbuf, 19, 1)) << 24);
153 unless ($fz == $zl) {
154 print("WARNING: rerenderfile: file '$fpth' is invalid - zsize $fz != $zl - skipping\n");
155 return;
156 }
157 print("Sending rendering request for z=$zl x=$calcx y=$calcy to regenerate '$fpth'\n");
158 if (system("$rrs $zl $calcx $calcy")) {
159 print("Error executing $rrs $zl $calcx $calcy\n");
160 }
161}
162
163# Par. 0: x
164# Par. 1: y
165sub calcpathfromcomponents($$) {
166 my @res = ();
167 my $i; my $x = $_[0]; my $y = $_[1];
168 for ($i = 4; $i >= 0; $i--) {
169 $res[$i] = sprintf("%d", (($x & 0x0f) << 4) + ($y & 0x0f));
170 $x = $x >> 4;
171 $y = $y >> 4;
172 }
173 return $res[0] . "/" . $res[1] . "/" . $res[2] . "/" . $res[3] . "/" . $res[4];
174}
175
176sub dorerenderexpiredlist() {
177 my $ll;
178 %rerenderlist = ();
179 while ($ll = <STDIN>) {
180 if ($ll =~ m!^(\d+)/(\d+)/(\d+)$!) {
181 my $x = $2; my $y = $3; my $z = $1;
182 #print("Handling z=$z x=$x y=$y\n");
183 if ($z != ($zoommax - 3)) {
184 print("Ignoring z=$z x=$x y=$y because of wrong zoom\n");
185 next;
186 }
187 my $cz; my $cx; my $cy; my $curz;
188 for ($curz = $zoommin; $curz <= $zoommax; $curz++) {
189 $cz = $z; $cx = $x; $cy = $y;
190 while ($cz < $curz) {
191 $cz++; $cx <<= 1; $cy <<= 1;
192 }
193 while ($cz > $curz) {
194 $cz--; $cx >>= 1; $cy >>= 1;
195 }
196 #print("Matching tile at z=$cz: x=$cx y=$cy -");
197 $cx = $cx & 0xfff8; $cy = $cy & 0xfff8;
198 #print(" rounded to x=$cx y=$cy\n");
199 $rerenderlist{$cz}{$cx}{$cy} = 1;
200 }
201 }
202 }
203 $filesdone = 0; $filesseen = 0;
204 foreach $z (sort(keys(%rerenderlist))) {
205 foreach $x (sort(keys(%{$rerenderlist{$z}}))) {
206 foreach $y (sort(keys(%{$rerenderlist{$z}{$x}}))) {
207 my $p = ${fspath} . '/' . $z . '/' . calcpathfromcomponents($x, $y) . '.meta';
208 #print("Checking: $z $x $y - Path: $p\n");
209 $filesseen++;
210 if (-e $p) {
211 $filesdone++;
212 print("Sending rendering request for z=$z x=$x y=$y to regenerate '$p'\n");
213 if (system("$rrs $z $x $y")) {
214 print("Error executing $rrs $z $x $y\n");
215 }
216 }
217 }
218 }
219 }
220 print("Sent re-rendering-requests for $filesdone files that actually existed (of $filesseen candidates)\n");
221}
222
223$fspath = '';
224for ($i = 0; $i < @ARGV; $i++) {
225 $curarg = $ARGV[$i];
226 if ($curarg eq '--help') {
227 print("Syntax: $0 [--help] [--zoom z] [--limit n] [--action a] [--rmemptydirs] [--rrs path] [--minage hrs] directory\n");
228 print(" --zoom z[-z2]: Zoom level(s) to handle (default: 0-20).\n");
229 print(" --limit n: Limit to the n oldest files (default: no limit)\n");
230 print(" --action what: what action to perform with the found files.\n");
231 print(" valid actions are: print delete rerender rerenderexpiredlist (default: print)\n");
232 print(" rerenderexpiredlist: this is very different from all other commands, as it\n");
233 print(" expects to read a list of expired tiles to rerender on STDIN. The list has\n");
234 print(" to be in the format that osm2pgsql spits out.\n");
235 print(" --rmemptydirs: remove empty subdirectories\n");
236 print(" --rrs path: if action is rerender, this gives the path to the rerender script.\n");
237 print(" it gets called with three parameters: z x y\n");
238 print(" --minage hrs: only care about files that are at least hrs hours old\n");
239 exit(1);
240 } elsif ($curarg eq '--zoom') {
241 $i++;
242 if ($i >= @ARGV) {
243 print("Error: --zoom requires a parameter\n");
244 exit(1);
245 } else {
246 if ($ARGV[$i] =~ m/^(\d+)-(\d+)$/) {
247 $zoommin = $1; $zoommax = $2;
248 } elsif ($ARGV[$i] =~ m/^(\d+)$/) {
249 $zoommin = $1; $zoommax = $1;
250 } else {
251 print("Error: --zoom needs a single numeric parameter or a range (e.g. 4 or 10-12)\n");
252 exit(1);
253 }
254 }
255 } elsif ($curarg eq '--limit') {
256 $i++;
257 if ($i >= @ARGV) {
258 print("Error: --limit requires a parameter\n");
259 exit(1);
260 } else {
261 if ($ARGV[$i] =~ m/^(\d+)$/) {
262 $filelimit = $1;
263 } else {
264 print("Error: --limit requires an positive integer as parameter.\n");
265 }
266 }
267 } elsif ($curarg eq '--action') {
268 $i++;
269 if ($i >= @ARGV) {
270 print("Error: --action requires a parameter\n");
271 exit(1);
272 } else {
273 if ($ARGV[$i] eq 'print') {
274 $action = 0;
275 } elsif ($ARGV[$i] eq 'delete') {
276 $action = 1;
277 } elsif ($ARGV[$i] eq 'rerender') {
278 $action = 2;
279 } elsif ($ARGV[$i] eq 'rerenderexpiredlist') {
280 $action = 3;
281 } else {
282 print("Error: Invalid action selected.\n");
283 exit(1);
284 }
285 }
286 } elsif ($curarg eq '--rrs') {
287 $i++;
288 if ($i >= @ARGV) {
289 print("Error: --rrs requires a parameter\n");
290 exit(1);
291 } else {
292 $rrs = $ARGV[$i];
293 }
294 } elsif ($curarg eq '--minage') {
295 $i++;
296 if ($i >= @ARGV) {
297 print("Error: --minage requires a parameter\n");
298 exit(1);
299 } else {
300 if ($ARGV[$i] =~ m/^(\d+)$/) {
301 $minage = $1;
302 } else {
303 print("Error: --minage requires an positive integer as parameter.\n");
304 }
305 }
306 } elsif ($curarg eq '--rmemptydirs') {
307 $rmemptydirs = 1;
308 } else {
309 if ($fspath eq '') {
310 $fspath = $curarg;
311 } else {
312 print("Too many parameters, or parameter(s) not understood.\n");
313 print("I can only handle one directory parameter, but I consider both ");
314 print("'$curarg' and '$fspath' a pathname since they are not a known parameter.\n");
315 exit(1);
316 }
317 }
318}
319if ($fspath eq '') {
320 print("ERROR: No path to clear given.\n");
321 exit(1);
322}
323if ($action == 3) { # This significantly differs from the rest of our operations
324 dorerenderexpiredlist();
325 exit(0);
326}
327@allentries = ();
328$totalfilesseen = 0;
329runrecursive($fspath, 0);
330@allentries = sort(sortbydateasc @allentries);
331print(int(@allentries)." files seen\n");
332$filesdone = 0;
333foreach $ent (@allentries) {
334 if ($filelimit > 0) {
335 if ($filesdone >= $filelimit) {
336 print("File limit $filelimit reached, exiting.\n");
337 exit(0);
338 }
339 }
340 #print("Handling File '$ent->[0]', $ent->[5] blocks [u: $ent->[3] g: $ent->[4] mtime: ".strftime("%Y-%m-%d.%H:%M:%S", localtime($ent->[1]))."]\n");
341 if ($action == 0) { # Print
342 print("$ent->[0]\n");
343 } elsif ($action == 1) { # Delete
344 print("DELETING $ent->[0] (mtime " . strftime("%Y-%m-%d.%H:%M:%S", localtime($ent->[1])) . ")\n");
345 unless (unlink($ent->[0])) {
346 print("ERROR: rm for $ent->[0] failed!\n");
347 }
348 } elsif ($action == 2) { # Rerender
349 rerenderfile($ent->[0]);
350 }
351 $filesdone++;
352}
This page took 0.159059 seconds and 4 git commands to generate.