Commit | Line | Data |
---|---|---|
ecf590ae MW |
1 | <?xml version="1.0" encoding="utf-8" ?> |
2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> | |
3 | <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> | |
4 | <head> | |
5 | <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> | |
6 | <meta name="generator" content="Docutils 0.12: http://docutils.sourceforge.net/" /> | |
7 | <title>LBM Benchmark Kernels Documentation</title> | |
8 | <style type="text/css"> | |
9 | ||
10 | /* | |
11 | :Author: David Goodger (goodger@python.org) | |
12 | :Id: $Id: html4css1.css 7614 2013-02-21 15:55:51Z milde $ | |
13 | :Copyright: This stylesheet has been placed in the public domain. | |
14 | ||
15 | Default cascading style sheet for the HTML output of Docutils. | |
16 | ||
17 | See http://docutils.sf.net/docs/howto/html-stylesheets.html for how to | |
18 | customize this style sheet. | |
19 | */ | |
20 | ||
21 | /* used to remove borders from tables and images */ | |
22 | .borderless, table.borderless td, table.borderless th { | |
23 | border: 0 } | |
24 | ||
25 | table.borderless td, table.borderless th { | |
26 | /* Override padding for "table.docutils td" with "! important". | |
27 | The right padding separates the table cells. */ | |
28 | padding: 0 0.5em 0 0 ! important } | |
29 | ||
30 | .first { | |
31 | /* Override more specific margin styles with "! important". */ | |
32 | margin-top: 0 ! important } | |
33 | ||
34 | .last, .with-subtitle { | |
35 | margin-bottom: 0 ! important } | |
36 | ||
37 | .hidden { | |
38 | display: none } | |
39 | ||
40 | a.toc-backref { | |
41 | text-decoration: none ; | |
42 | color: black } | |
43 | ||
44 | blockquote.epigraph { | |
45 | margin: 2em 5em ; } | |
46 | ||
47 | dl.docutils dd { | |
48 | margin-bottom: 0.5em } | |
49 | ||
50 | object[type="image/svg+xml"], object[type="application/x-shockwave-flash"] { | |
51 | overflow: hidden; | |
52 | } | |
53 | ||
54 | /* Uncomment (and remove this text!) to get bold-faced definition list terms | |
55 | dl.docutils dt { | |
56 | font-weight: bold } | |
57 | */ | |
58 | ||
59 | div.abstract { | |
60 | margin: 2em 5em } | |
61 | ||
62 | div.abstract p.topic-title { | |
63 | font-weight: bold ; | |
64 | text-align: center } | |
65 | ||
66 | div.admonition, div.attention, div.caution, div.danger, div.error, | |
67 | div.hint, div.important, div.note, div.tip, div.warning { | |
68 | margin: 2em ; | |
69 | border: medium outset ; | |
70 | padding: 1em } | |
71 | ||
72 | div.admonition p.admonition-title, div.hint p.admonition-title, | |
73 | div.important p.admonition-title, div.note p.admonition-title, | |
74 | div.tip p.admonition-title { | |
75 | font-weight: bold ; | |
76 | font-family: sans-serif } | |
77 | ||
78 | div.attention p.admonition-title, div.caution p.admonition-title, | |
79 | div.danger p.admonition-title, div.error p.admonition-title, | |
80 | div.warning p.admonition-title, .code .error { | |
81 | color: red ; | |
82 | font-weight: bold ; | |
83 | font-family: sans-serif } | |
84 | ||
85 | /* Uncomment (and remove this text!) to get reduced vertical space in | |
86 | compound paragraphs. | |
87 | div.compound .compound-first, div.compound .compound-middle { | |
88 | margin-bottom: 0.5em } | |
89 | ||
90 | div.compound .compound-last, div.compound .compound-middle { | |
91 | margin-top: 0.5em } | |
92 | */ | |
93 | ||
94 | div.dedication { | |
95 | margin: 2em 5em ; | |
96 | text-align: center ; | |
97 | font-style: italic } | |
98 | ||
99 | div.dedication p.topic-title { | |
100 | font-weight: bold ; | |
101 | font-style: normal } | |
102 | ||
103 | div.figure { | |
104 | margin-left: 2em ; | |
105 | margin-right: 2em } | |
106 | ||
107 | div.footer, div.header { | |
108 | clear: both; | |
109 | font-size: smaller } | |
110 | ||
111 | div.line-block { | |
112 | display: block ; | |
113 | margin-top: 1em ; | |
114 | margin-bottom: 1em } | |
115 | ||
116 | div.line-block div.line-block { | |
117 | margin-top: 0 ; | |
118 | margin-bottom: 0 ; | |
119 | margin-left: 1.5em } | |
120 | ||
121 | div.sidebar { | |
122 | margin: 0 0 0.5em 1em ; | |
123 | border: medium outset ; | |
124 | padding: 1em ; | |
125 | background-color: #ffffee ; | |
126 | width: 40% ; | |
127 | float: right ; | |
128 | clear: right } | |
129 | ||
130 | div.sidebar p.rubric { | |
131 | font-family: sans-serif ; | |
132 | font-size: medium } | |
133 | ||
134 | div.system-messages { | |
135 | margin: 5em } | |
136 | ||
137 | div.system-messages h1 { | |
138 | color: red } | |
139 | ||
140 | div.system-message { | |
141 | border: medium outset ; | |
142 | padding: 1em } | |
143 | ||
144 | div.system-message p.system-message-title { | |
145 | color: red ; | |
146 | font-weight: bold } | |
147 | ||
148 | div.topic { | |
149 | margin: 2em } | |
150 | ||
151 | h1.section-subtitle, h2.section-subtitle, h3.section-subtitle, | |
152 | h4.section-subtitle, h5.section-subtitle, h6.section-subtitle { | |
153 | margin-top: 0.4em } | |
154 | ||
155 | h1.title { | |
156 | text-align: center } | |
157 | ||
158 | h2.subtitle { | |
159 | text-align: center } | |
160 | ||
161 | hr.docutils { | |
162 | width: 75% } | |
163 | ||
164 | img.align-left, .figure.align-left, object.align-left { | |
165 | clear: left ; | |
166 | float: left ; | |
167 | margin-right: 1em } | |
168 | ||
169 | img.align-right, .figure.align-right, object.align-right { | |
170 | clear: right ; | |
171 | float: right ; | |
172 | margin-left: 1em } | |
173 | ||
174 | img.align-center, .figure.align-center, object.align-center { | |
175 | display: block; | |
176 | margin-left: auto; | |
177 | margin-right: auto; | |
178 | } | |
179 | ||
180 | .align-left { | |
181 | text-align: left } | |
182 | ||
183 | .align-center { | |
184 | clear: both ; | |
185 | text-align: center } | |
186 | ||
187 | .align-right { | |
188 | text-align: right } | |
189 | ||
190 | /* reset inner alignment in figures */ | |
191 | div.align-right { | |
192 | text-align: inherit } | |
193 | ||
194 | /* div.align-center * { */ | |
195 | /* text-align: left } */ | |
196 | ||
197 | ol.simple, ul.simple { | |
198 | margin-bottom: 1em } | |
199 | ||
200 | ol.arabic { | |
201 | list-style: decimal } | |
202 | ||
203 | ol.loweralpha { | |
204 | list-style: lower-alpha } | |
205 | ||
206 | ol.upperalpha { | |
207 | list-style: upper-alpha } | |
208 | ||
209 | ol.lowerroman { | |
210 | list-style: lower-roman } | |
211 | ||
212 | ol.upperroman { | |
213 | list-style: upper-roman } | |
214 | ||
215 | p.attribution { | |
216 | text-align: right ; | |
217 | margin-left: 50% } | |
218 | ||
219 | p.caption { | |
220 | font-style: italic } | |
221 | ||
222 | p.credits { | |
223 | font-style: italic ; | |
224 | font-size: smaller } | |
225 | ||
226 | p.label { | |
227 | white-space: nowrap } | |
228 | ||
229 | p.rubric { | |
230 | font-weight: bold ; | |
231 | font-size: larger ; | |
232 | color: maroon ; | |
233 | text-align: center } | |
234 | ||
235 | p.sidebar-title { | |
236 | font-family: sans-serif ; | |
237 | font-weight: bold ; | |
238 | font-size: larger } | |
239 | ||
240 | p.sidebar-subtitle { | |
241 | font-family: sans-serif ; | |
242 | font-weight: bold } | |
243 | ||
244 | p.topic-title { | |
245 | font-weight: bold } | |
246 | ||
247 | pre.address { | |
248 | margin-bottom: 0 ; | |
249 | margin-top: 0 ; | |
250 | font: inherit } | |
251 | ||
252 | pre.literal-block, pre.doctest-block, pre.math, pre.code { | |
253 | margin-left: 2em ; | |
254 | margin-right: 2em } | |
255 | ||
256 | pre.code .ln { color: grey; } /* line numbers */ | |
257 | pre.code, code { background-color: #eeeeee } | |
258 | pre.code .comment, code .comment { color: #5C6576 } | |
259 | pre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold } | |
260 | pre.code .literal.string, code .literal.string { color: #0C5404 } | |
261 | pre.code .name.builtin, code .name.builtin { color: #352B84 } | |
262 | pre.code .deleted, code .deleted { background-color: #DEB0A1} | |
263 | pre.code .inserted, code .inserted { background-color: #A3D289} | |
264 | ||
265 | span.classifier { | |
266 | font-family: sans-serif ; | |
267 | font-style: oblique } | |
268 | ||
269 | span.classifier-delimiter { | |
270 | font-family: sans-serif ; | |
271 | font-weight: bold } | |
272 | ||
273 | span.interpreted { | |
274 | font-family: sans-serif } | |
275 | ||
276 | span.option { | |
277 | white-space: nowrap } | |
278 | ||
279 | span.pre { | |
280 | white-space: pre } | |
281 | ||
282 | span.problematic { | |
283 | color: red } | |
284 | ||
285 | span.section-subtitle { | |
286 | /* font-size relative to parent (h1..h6 element) */ | |
287 | font-size: 80% } | |
288 | ||
289 | table.citation { | |
290 | border-left: solid 1px gray; | |
291 | margin-left: 1px } | |
292 | ||
293 | table.docinfo { | |
294 | margin: 2em 4em } | |
295 | ||
296 | table.docutils { | |
297 | margin-top: 0.5em ; | |
298 | margin-bottom: 0.5em } | |
299 | ||
300 | table.footnote { | |
301 | border-left: solid 1px black; | |
302 | margin-left: 1px } | |
303 | ||
304 | table.docutils td, table.docutils th, | |
305 | table.docinfo td, table.docinfo th { | |
306 | padding-left: 0.5em ; | |
307 | padding-right: 0.5em ; | |
308 | vertical-align: top } | |
309 | ||
310 | table.docutils th.field-name, table.docinfo th.docinfo-name { | |
311 | font-weight: bold ; | |
312 | text-align: left ; | |
313 | white-space: nowrap ; | |
314 | padding-left: 0 } | |
315 | ||
316 | /* "booktabs" style (no vertical lines) */ | |
317 | table.docutils.booktabs { | |
318 | border: 0px; | |
319 | border-top: 2px solid; | |
320 | border-bottom: 2px solid; | |
321 | border-collapse: collapse; | |
322 | } | |
323 | table.docutils.booktabs * { | |
324 | border: 0px; | |
325 | } | |
326 | table.docutils.booktabs th { | |
327 | border-bottom: thin solid; | |
328 | text-align: left; | |
329 | } | |
330 | ||
331 | h1 tt.docutils, h2 tt.docutils, h3 tt.docutils, | |
332 | h4 tt.docutils, h5 tt.docutils, h6 tt.docutils { | |
333 | font-size: 100% } | |
334 | ||
335 | ul.auto-toc { | |
336 | list-style-type: none } | |
337 | ||
e3f82424 MW |
338 | </style> |
339 | <style type="text/css"> | |
340 | ||
341 | ||
342 | h1, h2, h3, h4, h5, h6 { | |
343 | font-family: sans-serif; | |
344 | font-size: 100%; | |
345 | background-color: #dcdcdc; | |
346 | } | |
347 | ||
348 | h1.title { | |
349 | background-color: gray; | |
350 | color: white | |
351 | } | |
352 | ||
353 | table.footnote { | |
354 | padding-left: 0.5ex; | |
355 | } | |
356 | ||
357 | table.citation { | |
358 | padding-left: 0.5ex | |
359 | } | |
360 | ||
361 | td.label { | |
362 | width: 10%; | |
363 | } | |
364 | ||
365 | table, table.docutils, td, th { | |
366 | border: 0; | |
367 | } | |
368 | ||
369 | table.citation, table.footnote { | |
370 | width: 100%; | |
371 | } | |
372 | ||
373 | th { | |
374 | background-color: lavender ; | |
375 | } | |
376 | ||
377 | tr:nth-child(even) { | |
378 | xxbackground-color: aliceblue; | |
379 | background-color: white; | |
380 | } | |
381 | tr:nth-child(odd) { | |
382 | xxbackground-color: lavender; | |
383 | background-color: whitesmoke; | |
384 | } | |
385 | ||
386 | ||
387 | ||
ecf590ae MW |
388 | </style> |
389 | </head> | |
390 | <body> | |
0fde6e45 MW |
391 | <div class="document"> |
392 | ||
393 | ||
394 | <div class="line-block"> | |
395 | <div class="line">Copyright</div> | |
396 | <div class="line-block"> | |
397 | <div class="line">Markus Wittmann, 2016-2018</div> | |
398 | <div class="line">RRZE, University of Erlangen-Nuremberg, Germany</div> | |
399 | <div class="line">markus.wittmann -at- fau.de or hpc -at- rrze.fau.de</div> | |
400 | <div class="line"><br /></div> | |
401 | <div class="line">Viktor Haag, 2016</div> | |
402 | <div class="line">LSS, University of Erlangen-Nuremberg, Germany</div> | |
403 | <div class="line"><br /></div> | |
8cafd9ea MW |
404 | <div class="line">Michael Hussnaetter, 2017-2018</div> |
405 | <div class="line">University of Erlangen-Nuremberg, Germany</div> | |
406 | <div class="line">michael.hussnaetter -at- fau.de</div> | |
407 | <div class="line"><br /></div> | |
0fde6e45 MW |
408 | </div> |
409 | <div class="line">This file is part of the Lattice Boltzmann Benchmark Kernels (LbmBenchKernels).</div> | |
410 | <div class="line"><br /></div> | |
411 | <div class="line">LbmBenchKernels is free software: you can redistribute it and/or modify</div> | |
412 | <div class="line">it under the terms of the GNU General Public License as published by</div> | |
413 | <div class="line">the Free Software Foundation, either version 3 of the License, or</div> | |
414 | <div class="line">(at your option) any later version.</div> | |
415 | <div class="line"><br /></div> | |
416 | <div class="line">LbmBenchKernels is distributed in the hope that it will be useful,</div> | |
417 | <div class="line">but WITHOUT ANY WARRANTY; without even the implied warranty of</div> | |
418 | <div class="line">MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the</div> | |
419 | <div class="line">GNU General Public License for more details.</div> | |
420 | <div class="line"><br /></div> | |
421 | <div class="line">You should have received a copy of the GNU General Public License</div> | |
422 | <div class="line">along with LbmBenchKernels. If not, see <<a class="reference external" href="http://www.gnu.org/licenses/">http://www.gnu.org/licenses/</a>>.</div> | |
423 | </div> | |
424 | <p><strong>LBM Benchmark Kernels Documentation</strong></p> | |
ecf590ae MW |
425 | <div class="contents topic" id="contents"> |
426 | <p class="topic-title first">Contents</p> | |
427 | <ul class="auto-toc simple"> | |
0095f461 MW |
428 | <li><a class="reference internal" href="#introduction" id="id5">1 Introduction</a></li> |
429 | <li><a class="reference internal" href="#compilation" id="id6">2 Compilation</a><ul class="auto-toc"> | |
430 | <li><a class="reference internal" href="#debug-and-verification" id="id7">2.1 Debug and Verification</a></li> | |
431 | <li><a class="reference internal" href="#release-and-verification" id="id8">2.2 Release and Verification</a></li> | |
432 | <li><a class="reference internal" href="#benchmarking" id="id9">2.3 Benchmarking</a></li> | |
433 | <li><a class="reference internal" href="#compilers" id="id10">2.4 Compilers</a></li> | |
0fde6e45 MW |
434 | <li><a class="reference internal" href="#floating-point-precision" id="id11">2.5 Floating Point Precision</a></li> |
435 | <li><a class="reference internal" href="#cleaning" id="id12">2.6 Cleaning</a></li> | |
436 | <li><a class="reference internal" href="#options-summary" id="id13">2.7 Options Summary</a></li> | |
ecf590ae MW |
437 | </ul> |
438 | </li> | |
0fde6e45 MW |
439 | <li><a class="reference internal" href="#invocation" id="id14">3 Invocation</a><ul class="auto-toc"> |
440 | <li><a class="reference internal" href="#command-line-parameters" id="id15">3.1 Command Line Parameters</a></li> | |
441 | <li><a class="reference internal" href="#kernels" id="id16">3.2 Kernels</a></li> | |
ecf590ae MW |
442 | </ul> |
443 | </li> | |
0fde6e45 MW |
444 | <li><a class="reference internal" href="#id2" id="id17">4 Benchmarking</a><ul class="auto-toc"> |
445 | <li><a class="reference internal" href="#intel-compiler" id="id18">4.1 Intel Compiler</a></li> | |
446 | <li><a class="reference internal" href="#pinning" id="id19">4.2 Pinning</a></li> | |
447 | <li><a class="reference internal" href="#general-remarks" id="id20">4.3 General Remarks</a></li> | |
448 | <li><a class="reference internal" href="#padding" id="id21">4.4 Padding</a></li> | |
e3f82424 MW |
449 | </ul> |
450 | </li> | |
0fde6e45 MW |
451 | <li><a class="reference internal" href="#geometries" id="id22">5 Geometries</a></li> |
452 | <li><a class="reference internal" href="#performance-results" id="id23">6 Performance Results</a><ul class="auto-toc"> | |
453 | <li><a class="reference internal" href="#machine-specifications" id="id24">6.1 Machine Specifications</a></li> | |
454 | <li><a class="reference internal" href="#single-socket-results" id="id25">6.2 Single Socket Results</a></li> | |
0095f461 MW |
455 | </ul> |
456 | </li> | |
457 | <li><a class="reference internal" href="#licence" id="id26">7 Licence</a></li> | |
458 | <li><a class="reference internal" href="#acknowledgements" id="id27">8 Acknowledgements</a></li> | |
459 | <li><a class="reference internal" href="#bibliography" id="id28">9 Bibliography</a></li> | |
ecf590ae MW |
460 | </ul> |
461 | </div> | |
0095f461 MW |
462 | <div class="section" id="introduction"> |
463 | <h1><a class="toc-backref" href="#id5">1 Introduction</a></h1> | |
464 | <p>The lattice Boltzmann (LBM) benchmark kernels are a collection of LBM kernel | |
465 | implementations.</p> | |
466 | <p><strong>AS SUCH THE LBM BENCHMARK KERNELS ARE NO FULLY EQUIPPED CFD SOLVER AND SOLELY | |
467 | SERVES THE PURPOSE OF STUDYING POSSIBLE PERFORMANCE OPTIMIZATIONS AND/OR | |
468 | EXPERIMENTS.</strong></p> | |
469 | <p>Currently all kernels utilize a D3Q19 discretization and the | |
470 | two-relaxation-time (TRT) collision operator <a class="citation-reference" href="#ginzburg-2008" id="id1">[ginzburg-2008]</a>. | |
0fde6e45 | 471 | All operations are carried out in double or single precision arithmetic.</p> |
0095f461 | 472 | </div> |
ecf590ae | 473 | <div class="section" id="compilation"> |
0095f461 | 474 | <h1><a class="toc-backref" href="#id6">2 Compilation</a></h1> |
ecf590ae MW |
475 | <p>The benchmark framework currently supports only Linux systems and the GCC and |
476 | Intel compilers. Every other configuration probably requires adjustment inside | |
0095f461 | 477 | the code and the makefiles. Furthermore some code might be platform or at least |
ecf590ae MW |
478 | POSIX specific.</p> |
479 | <p>The benchmark can be build via <tt class="docutils literal">make</tt> from the <tt class="docutils literal">src</tt> subdirectory. This will | |
480 | generate one binary which hosts all implemented benchmark kernels.</p> | |
481 | <p>Binaries are located under the <tt class="docutils literal">bin</tt> subdirectory and will have different names | |
482 | depending on compiler and build configuration.</p> | |
0095f461 MW |
483 | <p>Compilation can target debug or release builds. Combined with both build types |
484 | verification can be enabled, which increases the runtime and hence is not | |
485 | suited for benchmarking.</p> | |
ecf590ae | 486 | <div class="section" id="debug-and-verification"> |
0095f461 | 487 | <h2><a class="toc-backref" href="#id7">2.1 Debug and Verification</a></h2> |
ecf590ae | 488 | <pre class="literal-block"> |
e3f82424 | 489 | make BUILD=debug BENCHMARK=off |
ecf590ae | 490 | </pre> |
e3f82424 | 491 | <p>Running <tt class="docutils literal">make</tt> with <tt class="docutils literal">BUILD=debug</tt> builds the debug version of |
ecf590ae MW |
492 | the benchmark kernels, where no optimizations are performed, line numbers and |
493 | debug symbols are included as well as <tt class="docutils literal">DEBUG</tt> will be defined. The resulting | |
494 | binary will be found in the <tt class="docutils literal">bin</tt> subdirectory and named | |
495 | <tt class="docutils literal"><span class="pre">lbmbenchk-linux-<compiler>-debug</span></tt>.</p> | |
e3f82424 MW |
496 | <p>Specifying <tt class="docutils literal">BENCHMARK=off</tt> turns on verification |
497 | (<tt class="docutils literal">VERIFICATION=on</tt>), statistics (<tt class="docutils literal">STATISTICS=on</tt>), and VTK output | |
ecf590ae MW |
498 | (<tt class="docutils literal">VTK_OUTPUT=on</tt>) enabled.</p> |
499 | <p>Please note that the generated binary will therefore | |
500 | exhibit a poor performance.</p> | |
501 | </div> | |
0095f461 MW |
502 | <div class="section" id="release-and-verification"> |
503 | <h2><a class="toc-backref" href="#id8">2.2 Release and Verification</a></h2> | |
504 | <p>Verification with the debug builds can be extremely slow. Hence verification | |
505 | capabilities can be build with release builds:</p> | |
506 | <pre class="literal-block"> | |
507 | make BENCHMARK=off | |
508 | </pre> | |
509 | </div> | |
ecf590ae | 510 | <div class="section" id="benchmarking"> |
0095f461 | 511 | <h2><a class="toc-backref" href="#id9">2.3 Benchmarking</a></h2> |
ecf590ae MW |
512 | <p>To generate a binary for benchmarking run make with</p> |
513 | <pre class="literal-block"> | |
e3f82424 | 514 | make |
ecf590ae | 515 | </pre> |
e3f82424 | 516 | <p>As default <tt class="docutils literal">BENCHMARK=on</tt> and <tt class="docutils literal">BUILD=release</tt> is set, where |
0095f461 | 517 | <tt class="docutils literal">BUILD=release</tt> turns optimizations on and <tt class="docutils literal">BENCHMARK=on</tt> disables |
ecf590ae | 518 | verfification, statistics, and VTK output.</p> |
0095f461 MW |
519 | <p>See Options Summary below for further description of options which can be |
520 | applied, e.g. TARCH as well as the Benchmarking section.</p> | |
ecf590ae MW |
521 | </div> |
522 | <div class="section" id="compilers"> | |
0095f461 | 523 | <h2><a class="toc-backref" href="#id10">2.4 Compilers</a></h2> |
ecf590ae MW |
524 | <p>Currently only the GCC and Intel compiler under Linux are supported. Between |
525 | both configuration can be chosen via <tt class="docutils literal"><span class="pre">CONFIG=linux-gcc</span></tt> or | |
526 | <tt class="docutils literal"><span class="pre">CONFIG=linux-intel</span></tt>.</p> | |
527 | </div> | |
0fde6e45 MW |
528 | <div class="section" id="floating-point-precision"> |
529 | <h2><a class="toc-backref" href="#id11">2.5 Floating Point Precision</a></h2> | |
530 | <p>As default double precision data types are used for storing PDFs and floating | |
531 | point constants. Furthermore, this is the default for the intrincis kernels. | |
532 | With the <tt class="docutils literal">PRECISION=sp</tt> variable this can be changed to single precision.</p> | |
533 | <pre class="literal-block"> | |
534 | make PRECISION=sp # build for single precision kernels | |
535 | ||
536 | make PRECISION=dp # build for double precision kernels (defalt) | |
537 | </pre> | |
538 | </div> | |
e3f82424 | 539 | <div class="section" id="cleaning"> |
0fde6e45 | 540 | <h2><a class="toc-backref" href="#id12">2.6 Cleaning</a></h2> |
e3f82424 MW |
541 | <p>For each configuration and build (debug/release) a subdirectory under the |
542 | <tt class="docutils literal">src/obj</tt> directory is created where the dependency and object files are | |
543 | stored. | |
544 | With</p> | |
545 | <pre class="literal-block"> | |
546 | make CONFIG=... BUILD=... clean | |
547 | </pre> | |
548 | <p>a specific combination is select and cleaned, whereas with</p> | |
549 | <pre class="literal-block"> | |
550 | make clean-all | |
551 | </pre> | |
552 | <p>all object and dependency files are deleted.</p> | |
553 | </div> | |
ecf590ae | 554 | <div class="section" id="options-summary"> |
0fde6e45 | 555 | <h2><a class="toc-backref" href="#id13">2.7 Options Summary</a></h2> |
0095f461 | 556 | <p>Options that can be specified when building the suite with make:</p> |
ecf590ae MW |
557 | <table border="1" class="docutils"> |
558 | <colgroup> | |
ecf590ae | 559 | <col width="7%" /> |
0095f461 MW |
560 | <col width="12%" /> |
561 | <col width="6%" /> | |
562 | <col width="75%" /> | |
ecf590ae | 563 | </colgroup> |
0095f461 MW |
564 | <thead valign="bottom"> |
565 | <tr><th class="head">name</th> | |
566 | <th class="head">values</th> | |
567 | <th class="head">default</th> | |
568 | <th class="head">description</th> | |
ecf590ae | 569 | </tr> |
0095f461 MW |
570 | </thead> |
571 | <tbody valign="top"> | |
ecf590ae MW |
572 | <tr><td>BENCHMARK</td> |
573 | <td>on, off</td> | |
e3f82424 MW |
574 | <td>on</td> |
575 | <td>If enabled, disables VERIFICATION, STATISTICS, VTK_OUTPUT. If disabled enables the three former options.</td> | |
ecf590ae MW |
576 | </tr> |
577 | <tr><td>BUILD</td> | |
578 | <td>debug, release</td> | |
e3f82424 | 579 | <td>release</td> |
0095f461 | 580 | <td>debug: no optimization, debug symbols, DEBUG defined. release: optimizations enabled.</td> |
ecf590ae MW |
581 | </tr> |
582 | <tr><td>CONFIG</td> | |
583 | <td>linux-gcc, linux-intel</td> | |
584 | <td>linux-intel</td> | |
585 | <td>Select GCC or Intel compiler.</td> | |
586 | </tr> | |
587 | <tr><td>ISA</td> | |
8cafd9ea | 588 | <td>avx512, avx, sse</td> |
ecf590ae | 589 | <td>avx</td> |
0095f461 | 590 | <td>Determines which ISA extension is used for macro definitions of the intrinsics. This is <em>not</em> the architecture the compiler generates code for.</td> |
ecf590ae MW |
591 | </tr> |
592 | <tr><td>OPENMP</td> | |
593 | <td>on, off</td> | |
594 | <td>on</td> | |
595 | <td>OpenMP, i.,e.. threading support.</td> | |
596 | </tr> | |
0fde6e45 MW |
597 | <tr><td>PRECISION</td> |
598 | <td>dp, sp</td> | |
599 | <td>dp</td> | |
600 | <td>Floating point precision used for data type, arithmetic, and intrincics.</td> | |
601 | </tr> | |
ecf590ae MW |
602 | <tr><td>STATISTICS</td> |
603 | <td>on, off</td> | |
604 | <td>off</td> | |
605 | <td>View statistics, like density etc, during simulation.</td> | |
606 | </tr> | |
e3f82424 MW |
607 | <tr><td>TARCH</td> |
608 | <td>--</td> | |
609 | <td>--</td> | |
610 | <td>Via TARCH the architecture the compiler generates code for can be overridden. The value depends on the chosen compiler.</td> | |
611 | </tr> | |
ecf590ae MW |
612 | <tr><td>VERIFICATION</td> |
613 | <td>on, off</td> | |
614 | <td>off</td> | |
615 | <td>Turn verification on/off.</td> | |
616 | </tr> | |
617 | <tr><td>VTK_OUTPUT</td> | |
618 | <td>on, off</td> | |
619 | <td>off</td> | |
620 | <td>Enable/Disable VTK file output.</td> | |
621 | </tr> | |
622 | </tbody> | |
623 | </table> | |
8cafd9ea MW |
624 | <p><strong>Suboptions for ``ISA=avx512``</strong></p> |
625 | <table border="1" class="docutils"> | |
626 | <colgroup> | |
627 | <col width="20%" /> | |
628 | <col width="5%" /> | |
629 | <col width="5%" /> | |
630 | <col width="69%" /> | |
631 | </colgroup> | |
632 | <thead valign="bottom"> | |
633 | <tr><th class="head">name</th> | |
634 | <th class="head">values</th> | |
635 | <th class="head">default</th> | |
636 | <th class="head">description</th> | |
637 | </tr> | |
638 | </thead> | |
639 | <tbody valign="top"> | |
640 | <tr><td>SOFTWARE_PREFETCH_LOOKAHEAD_L1</td> | |
641 | <td>int >= 0</td> | |
642 | <td>0</td> | |
643 | <td>Software prefetch lookahead of elements into L1 cache, value is multiplied by vector size (<tt class="docutils literal">VSIZE</tt>).</td> | |
644 | </tr> | |
645 | <tr><td>SOFTWARE_PREFETCH_LOOKAHEAD_L2</td> | |
646 | <td>int >= 0</td> | |
647 | <td>0</td> | |
648 | <td>Software prefetch lookahead of elements into L2 cache, value is multiplied by vector size (<tt class="docutils literal">VSIZE</tt>).</td> | |
649 | </tr> | |
650 | </tbody> | |
651 | </table> | |
652 | <p>Please note this options require AVX-512 PF support of the target processor.</p> | |
ecf590ae MW |
653 | </div> |
654 | </div> | |
655 | <div class="section" id="invocation"> | |
0fde6e45 | 656 | <h1><a class="toc-backref" href="#id14">3 Invocation</a></h1> |
ecf590ae | 657 | <p>Running the binary will print among the GPL licence header a line like the following:</p> |
e3f82424 MW |
658 | <pre class="literal-block"> |
659 | LBM Benchmark Kernels 0.1, compiled Jul 5 2017 21:59:22, type: verification | |
660 | </pre> | |
ecf590ae | 661 | <p>if verfication was enabled during compilation or</p> |
e3f82424 MW |
662 | <pre class="literal-block"> |
663 | LBM Benchmark Kernels 0.1, compiled Jul 5 2017 21:59:22, type: benchmark | |
664 | </pre> | |
ecf590ae MW |
665 | <p>if verfication was disabled during compilation.</p> |
666 | <div class="section" id="command-line-parameters"> | |
0fde6e45 | 667 | <h2><a class="toc-backref" href="#id15">3.1 Command Line Parameters</a></h2> |
ecf590ae MW |
668 | <p>Running the binary with <tt class="docutils literal"><span class="pre">-h</span></tt> list all available parameters:</p> |
669 | <pre class="literal-block"> | |
670 | Usage: | |
671 | ./lbmbenchk -list | |
672 | ./lbmbenchk | |
8cafd9ea | 673 | [-dims XxYxZ] [-geometry box|channel|pipe|blocks[-<block size>]] [-iterations <iterations>] [-lattice-dump-ascii] |
ecf590ae MW |
674 | [-rho-in <density>] [-rho-out <density] [-omega <omega>] [-kernel <kernel>] |
675 | [-periodic-x] | |
676 | [-t <number of threads>] | |
677 | [-pin core{,core}*] | |
678 | [-verify] | |
679 | -- <kernel specific parameters> | |
680 | ||
681 | -list List available kernels. | |
682 | ||
683 | -dims XxYxZ Specify geometry dimensions. | |
684 | ||
685 | -geometry blocks-<block size> | |
686 | Geometetry with blocks of size <block size> regularily layout out. | |
687 | </pre> | |
688 | <p>If an option is specified multiple times the last one overrides previous ones. | |
689 | This holds also true for <tt class="docutils literal"><span class="pre">-verify</span></tt> which sets geometry dimensions, | |
690 | iterations, etc, which can afterward be override, e.g.:</p> | |
691 | <pre class="literal-block"> | |
0fde6e45 | 692 | $ bin/lbmbenchk-linux-intel-release-dp -verfiy -dims 32x32x32 |
ecf590ae | 693 | </pre> |
0095f461 | 694 | <p>Kernel specific parameters can be obtained via selecting the specific kernel |
ecf590ae MW |
695 | and passing <tt class="docutils literal"><span class="pre">-h</span></tt> as parameter:</p> |
696 | <pre class="literal-block"> | |
0fde6e45 | 697 | $ bin/lbmbenchk-linux-intel-release-dp -kernel kernel-name -- -h |
ecf590ae MW |
698 | ... |
699 | Kernel parameters: | |
700 | [-blk <n>] [-blk-[xyz] <n>] | |
701 | </pre> | |
702 | <p>A list of all available kernels can be obtained via <tt class="docutils literal"><span class="pre">-list</span></tt>:</p> | |
703 | <pre class="literal-block"> | |
0fde6e45 | 704 | $ ../bin/lbmbenchk-linux-gcc-debug-dp -list |
ecf590ae MW |
705 | Lattice Boltzmann Benchmark Kernels (LbmBenchKernels) Copyright (C) 2016, 2017 LSS, RRZE |
706 | This program comes with ABSOLUTELY NO WARRANTY; for details see LICENSE. | |
707 | This is free software, and you are welcome to redistribute it under certain conditions. | |
708 | ||
709 | LBM Benchmark Kernels 0.1, compiled Jul 5 2017 21:59:22, type: verification | |
710 | Available kernels to benchmark: | |
711 | list-aa-pv-soa | |
712 | list-aa-ria-soa | |
713 | list-aa-soa | |
714 | list-aa-aos | |
715 | list-pull-split-nt-1s-soa | |
716 | list-pull-split-nt-2s-soa | |
717 | list-push-soa | |
718 | list-push-aos | |
719 | list-pull-soa | |
720 | list-pull-aos | |
721 | push-soa | |
722 | push-aos | |
723 | pull-soa | |
724 | pull-aos | |
725 | blk-push-soa | |
726 | blk-push-aos | |
727 | blk-pull-soa | |
728 | blk-pull-aos | |
729 | </pre> | |
730 | </div> | |
e3f82424 | 731 | <div class="section" id="kernels"> |
0fde6e45 | 732 | <h2><a class="toc-backref" href="#id16">3.2 Kernels</a></h2> |
e3f82424 MW |
733 | <p>The following list shortly describes available kernels:</p> |
734 | <ul class="simple"> | |
0fde6e45 | 735 | <li><strong>push-soa/push-aos/pull-soa/pull-aos</strong>: |
e3f82424 MW |
736 | Unoptimized kernels (but stream/collide are already fused) using two grids as |
737 | source and destination. Implement push/pull semantics as well structure of | |
738 | arrays (soa) or array of structures (aos) layout.</li> | |
0fde6e45 | 739 | <li><strong>blk-push-soa/blk-push-aos/blk-pull-soa/blk-pull-aos</strong>: |
e3f82424 MW |
740 | The same as the unoptimized kernels without the blk prefix, except that they support |
741 | spatial blocking, i.e. loop blocking of the three loops used to iterate over | |
742 | the lattice. Here manual work sharing for OpenMP is used.</li> | |
0fde6e45 MW |
743 | <li><strong>aa-aos/aa-soa</strong>: |
744 | Straight forward implementation of AA pattern on full array with blocking support. | |
745 | Manual work sharing for OpenMP is used. Domain is partitioned only along the x dimension.</li> | |
746 | <li><strong>aa-vec-soa/aa-vec-sl-soa</strong>: | |
747 | Optimized AA kernel with intrinsics on full array. aa-vec-sl-soa uses only | |
748 | one loop for iterating over the lattice instead of three nested ones.</li> | |
749 | <li><strong>list-push-soa/list-push-aos/list-pull-soa/list-pull-aos</strong>: | |
e3f82424 MW |
750 | The same as the unoptimized kernels without the list prefix, but for indirect addressing. |
751 | Here only a 1D vector of is used to store the fluid nodes, omitting the | |
752 | obstacles. An adjacency list is used to recover the neighborhood associations.</li> | |
0fde6e45 | 753 | <li><strong>list-pull-split-nt-1s-soa/list-pull-split-nt-2s-soa</strong>: |
e3f82424 MW |
754 | Optimized variant of list-pull-soa. Chunks of the lattice are processed as |
755 | once. Postcollision values are written back via nontemporal stores in 18 (1s) | |
756 | or 9 (2s) loops.</li> | |
0fde6e45 | 757 | <li><strong>list-aa-aos/list-aa-soa</strong>: |
e3f82424 MW |
758 | Unoptimized implementation of the AA pattern for the 1D vector with adjacency |
759 | list. Supported are array of structures (aos) and structure of arrays (soa) | |
760 | data layout is supported.</li> | |
0fde6e45 | 761 | <li><strong>list-aa-ria-soa</strong>: |
e3f82424 MW |
762 | Implementation of AA pattern with intrinsics for the 1D vector with adjacency |
763 | list. Furthermore it contains a vectorized even time step and run length | |
764 | coding to reduce the loop balance of the odd time step.</li> | |
0fde6e45 | 765 | <li><strong>list-aa-pv-soa</strong>: |
e3f82424 MW |
766 | All optimizations of list-aa-ria-soa. Additional with partial vectorization |
767 | of the odd time step.</li> | |
768 | </ul> | |
769 | <p>Note that all array of structures (aos) kernels might require blocking | |
770 | (depending on the domain size) to reach the performance of their structure of | |
771 | arrays (soa) counter parts.</p> | |
772 | <p>The following table summarizes the properties of the kernels. Here <strong>D</strong> means | |
773 | direct addressing, i.e. full array, <strong>I</strong> means indirect addressing, i.e. 1D | |
774 | vector with adjacency list, <strong>x</strong> means supported, whereas <strong>--</strong> means unsupported. | |
0fde6e45 | 775 | The loop balance B_l is computed for D3Q19 model with <strong>double precision</strong> floating |
e3f82424 MW |
776 | point for PDFs (8 byte) and 4 byte integers for the index (adjacency list). |
777 | As list-aa-ria-soa and list-aa-pv-soa support run length coding their effective | |
778 | loop balance depends on the geometry. The effective loop balance is printed | |
779 | during each run.</p> | |
780 | <table border="1" class="docutils"> | |
781 | <colgroup> | |
782 | <col width="29%" /> | |
783 | <col width="14%" /> | |
784 | <col width="14%" /> | |
785 | <col width="6%" /> | |
786 | <col width="10%" /> | |
787 | <col width="10%" /> | |
788 | <col width="16%" /> | |
789 | </colgroup> | |
790 | <thead valign="bottom"> | |
791 | <tr><th class="head">kernel name</th> | |
792 | <th class="head">prop. step</th> | |
793 | <th class="head">data layout</th> | |
794 | <th class="head">addr.</th> | |
795 | <th class="head">parallel</th> | |
796 | <th class="head">blocking</th> | |
797 | <th class="head">B_l [B/FLUP]</th> | |
798 | </tr> | |
799 | </thead> | |
800 | <tbody valign="top"> | |
801 | <tr><td>push-soa</td> | |
802 | <td>OS</td> | |
803 | <td>SoA</td> | |
804 | <td>D</td> | |
805 | <td>x</td> | |
806 | <td>--</td> | |
807 | <td>456</td> | |
808 | </tr> | |
809 | <tr><td>push-aos</td> | |
810 | <td>OS</td> | |
811 | <td>AoS</td> | |
812 | <td>D</td> | |
813 | <td>x</td> | |
814 | <td>--</td> | |
815 | <td>456</td> | |
816 | </tr> | |
817 | <tr><td>pull-soa</td> | |
818 | <td>OS</td> | |
819 | <td>SoA</td> | |
820 | <td>D</td> | |
821 | <td>x</td> | |
822 | <td>--</td> | |
823 | <td>456</td> | |
824 | </tr> | |
825 | <tr><td>pull-aos</td> | |
826 | <td>OS</td> | |
827 | <td>AoS</td> | |
828 | <td>D</td> | |
829 | <td>x</td> | |
830 | <td>--</td> | |
831 | <td>456</td> | |
832 | </tr> | |
833 | <tr><td>blk-push-soa</td> | |
834 | <td>OS</td> | |
835 | <td>SoA</td> | |
836 | <td>D</td> | |
837 | <td>x</td> | |
838 | <td>x</td> | |
839 | <td>456</td> | |
840 | </tr> | |
841 | <tr><td>blk-push-aos</td> | |
842 | <td>OS</td> | |
843 | <td>AoS</td> | |
844 | <td>D</td> | |
845 | <td>x</td> | |
846 | <td>x</td> | |
847 | <td>456</td> | |
848 | </tr> | |
849 | <tr><td>blk-pull-soa</td> | |
850 | <td>OS</td> | |
851 | <td>SoA</td> | |
852 | <td>D</td> | |
853 | <td>x</td> | |
854 | <td>x</td> | |
855 | <td>456</td> | |
856 | </tr> | |
857 | <tr><td>blk-pull-aos</td> | |
858 | <td>OS</td> | |
859 | <td>AoS</td> | |
860 | <td>D</td> | |
861 | <td>x</td> | |
862 | <td>x</td> | |
863 | <td>456</td> | |
864 | </tr> | |
0fde6e45 MW |
865 | <tr><td>aa-soa</td> |
866 | <td>AA</td> | |
867 | <td>SoA</td> | |
868 | <td>D</td> | |
869 | <td>x</td> | |
870 | <td>x</td> | |
871 | <td>304</td> | |
872 | </tr> | |
873 | <tr><td>aa-aos</td> | |
874 | <td>AA</td> | |
875 | <td>AoS</td> | |
876 | <td>D</td> | |
877 | <td>x</td> | |
878 | <td>x</td> | |
879 | <td>304</td> | |
880 | </tr> | |
881 | <tr><td>aa-vec-soa</td> | |
882 | <td>AA</td> | |
883 | <td>SoA</td> | |
884 | <td>D</td> | |
885 | <td>x</td> | |
886 | <td>x</td> | |
887 | <td>304</td> | |
888 | </tr> | |
889 | <tr><td>aa-vec-sl-soa</td> | |
890 | <td>AA</td> | |
891 | <td>SoA</td> | |
892 | <td>D</td> | |
893 | <td>x</td> | |
894 | <td>x</td> | |
895 | <td>304</td> | |
896 | </tr> | |
e3f82424 MW |
897 | <tr><td>list-push-soa</td> |
898 | <td>OS</td> | |
899 | <td>SoA</td> | |
900 | <td>I</td> | |
901 | <td>x</td> | |
902 | <td>x</td> | |
903 | <td>528</td> | |
904 | </tr> | |
905 | <tr><td>list-push-aos</td> | |
906 | <td>OS</td> | |
907 | <td>AoS</td> | |
908 | <td>I</td> | |
909 | <td>x</td> | |
910 | <td>x</td> | |
911 | <td>528</td> | |
912 | </tr> | |
913 | <tr><td>list-pull-soa</td> | |
914 | <td>OS</td> | |
915 | <td>SoA</td> | |
916 | <td>I</td> | |
917 | <td>x</td> | |
918 | <td>x</td> | |
919 | <td>528</td> | |
920 | </tr> | |
921 | <tr><td>list-pull-aos</td> | |
922 | <td>OS</td> | |
923 | <td>AoS</td> | |
924 | <td>I</td> | |
925 | <td>x</td> | |
926 | <td>x</td> | |
927 | <td>528</td> | |
928 | </tr> | |
929 | <tr><td>list-pull-split-nt-1s</td> | |
930 | <td>OS</td> | |
931 | <td>SoA</td> | |
932 | <td>I</td> | |
933 | <td>x</td> | |
934 | <td>x</td> | |
935 | <td>376</td> | |
936 | </tr> | |
937 | <tr><td>list-pull-split-nt-2s</td> | |
938 | <td>OS</td> | |
939 | <td>SoA</td> | |
940 | <td>I</td> | |
941 | <td>x</td> | |
942 | <td>x</td> | |
943 | <td>376</td> | |
944 | </tr> | |
945 | <tr><td>list-aa-soa</td> | |
946 | <td>AA</td> | |
947 | <td>SoA</td> | |
948 | <td>I</td> | |
949 | <td>x</td> | |
950 | <td>x</td> | |
951 | <td>340</td> | |
952 | </tr> | |
953 | <tr><td>list-aa-aos</td> | |
954 | <td>AA</td> | |
955 | <td>AoS</td> | |
956 | <td>I</td> | |
957 | <td>x</td> | |
958 | <td>x</td> | |
959 | <td>340</td> | |
960 | </tr> | |
961 | <tr><td>list-aa-ria-soa</td> | |
962 | <td>AA</td> | |
963 | <td>SoA</td> | |
964 | <td>I</td> | |
965 | <td>x</td> | |
966 | <td>x</td> | |
967 | <td>304-342</td> | |
968 | </tr> | |
969 | <tr><td>list-aa-pv-soa</td> | |
970 | <td>AA</td> | |
971 | <td>SoA</td> | |
972 | <td>I</td> | |
973 | <td>x</td> | |
974 | <td>x</td> | |
975 | <td>304-342</td> | |
976 | </tr> | |
977 | </tbody> | |
978 | </table> | |
979 | </div> | |
ecf590ae | 980 | </div> |
0095f461 | 981 | <div class="section" id="id2"> |
0fde6e45 | 982 | <h1><a class="toc-backref" href="#id17">4 Benchmarking</a></h1> |
ecf590ae MW |
983 | <p>Correct benchmarking is a nontrivial task. Whenever benchmark results should be |
984 | created make sure the binary was compiled with:</p> | |
985 | <ul class="simple"> | |
e3f82424 MW |
986 | <li><tt class="docutils literal">BENCHMARK=on</tt> (default if not overriden) and</li> |
987 | <li><tt class="docutils literal">BUILD=release</tt> (default if not overriden) and</li> | |
8cafd9ea | 988 | <li>the correct ISA for macros (i.e. intrinsics) is used, selected via <tt class="docutils literal">ISA</tt> and</li> |
ecf590ae MW |
989 | <li>use <tt class="docutils literal">TARCH</tt> to specify the architecture the compiler generates code for.</li> |
990 | </ul> | |
0095f461 | 991 | <div class="section" id="intel-compiler"> |
0fde6e45 | 992 | <h2><a class="toc-backref" href="#id18">4.1 Intel Compiler</a></h2> |
0095f461 MW |
993 | <p>For the Intel compiler one can specify depending on the target ISA extension:</p> |
994 | <ul class="simple"> | |
8cafd9ea | 995 | <li>SSE: <tt class="docutils literal"><span class="pre">TARCH=-xSSE4.2</span></tt></li> |
0095f461 MW |
996 | <li>AVX: <tt class="docutils literal"><span class="pre">TARCH=-xAVX</span></tt></li> |
997 | <li>AVX2 and FMA: <tt class="docutils literal"><span class="pre">TARCH=-xCORE-AVX2,-fma</span></tt></li> | |
998 | <li>AVX512: <tt class="docutils literal"><span class="pre">TARCH=-xCORE-AVX512</span></tt></li> | |
999 | <li>KNL: <tt class="docutils literal"><span class="pre">TARCH=-xMIC-AVX512</span></tt></li> | |
1000 | </ul> | |
1001 | <p>Compiling for an architecture supporting AVX (Sandy Bridge, Ivy Bridge):</p> | |
1002 | <pre class="literal-block"> | |
1003 | make ISA=avx TARCH=-xAVX | |
1004 | </pre> | |
1005 | <p>Compiling for an architecture supporting AVX2 (Haswell, Broadwell):</p> | |
1006 | <pre class="literal-block"> | |
1007 | make ISA=avx TARCH=-xCORE-AVX2,-fma | |
1008 | </pre> | |
1009 | <p>WARNING: ISA is here still set to <tt class="docutils literal">avx</tt> as currently we have the FMA intrinsics not | |
1010 | implemented. This might change in the future.</p> | |
8cafd9ea MW |
1011 | <!-- TODO: add isa=avx512 and add docu for knl --> |
1012 | <!-- TODO: kein prefetching wenn AVX-512 PF nicht unterstuetz wird --> | |
0095f461 MW |
1013 | <p>Compiling for an architecture supporting AVX-512 (Skylake):</p> |
1014 | <pre class="literal-block"> | |
8cafd9ea MW |
1015 | make ISA=avx512 TARCH=-xCORE-AVX512 |
1016 | </pre> | |
1017 | <p>Please note that for the AVX512 gather kernels software prefetching for the | |
1018 | gather instructions is disabled per default. | |
1019 | To enable it set <tt class="docutils literal">SOFTWARE_PREFETCH_LOOKAHEAD_L1</tt> and/or | |
1020 | <tt class="docutils literal">SOFTWARE_PREFETCH_LOOKAHEAD_L2</tt> to a value greater than <tt class="docutils literal">0</tt> during | |
1021 | compilation. Note that this requires AVX-512 PF support from the target | |
1022 | processor.</p> | |
1023 | <p>Compiling for MIC architecture KNL supporting AVX-512 and AVX-512 PF:</p> | |
1024 | <pre class="literal-block"> | |
1025 | make ISA=avx512 TARCH=-xMIC-AVX512 | |
1026 | </pre> | |
1027 | <p>or optionally with software prefetch enabled:</p> | |
1028 | <pre class="literal-block"> | |
1029 | make ISA=avx512 TARCH=-xMIC-AVX512 SOFTWARE_PREFETCH_LOOKAHEAD_L1=<value> SOFTWARE_PREFETCH_LOOKAHEAD_L2=<value> | |
0095f461 | 1030 | </pre> |
0095f461 MW |
1031 | </div> |
1032 | <div class="section" id="pinning"> | |
0fde6e45 | 1033 | <h2><a class="toc-backref" href="#id19">4.2 Pinning</a></h2> |
ecf590ae | 1034 | <p>During benchmarking pinning should be used via the <tt class="docutils literal"><span class="pre">-pin</span></tt> parameter. Running |
0095f461 | 1035 | a benchmark with 10 threads and pin them to the first 10 cores works like</p> |
ecf590ae | 1036 | <pre class="literal-block"> |
0fde6e45 | 1037 | $ bin/lbmbenchk-linux-intel-release-dp ... -t 10 -pin $(seq -s , 0 9) |
ecf590ae | 1038 | </pre> |
0095f461 MW |
1039 | </div> |
1040 | <div class="section" id="general-remarks"> | |
0fde6e45 | 1041 | <h2><a class="toc-backref" href="#id20">4.3 General Remarks</a></h2> |
0095f461 | 1042 | <p>Things the binary does nor check or control:</p> |
ecf590ae MW |
1043 | <ul class="simple"> |
1044 | <li>transparent huge pages: when allocating memory small 4 KiB pages might be | |
1045 | replaced with larger ones. This is in general a good thing, but if this is | |
e3f82424 MW |
1046 | really the case, depends on the system settings (check e.g. the status of |
1047 | <tt class="docutils literal">/sys/kernel/mm/transparent_hugepage/enabled</tt>). | |
1048 | Currently <tt class="docutils literal">madvise(MADV_HUGEPAGE)</tt> is used for allocations which are aligned to | |
1049 | a 4 KiB page, which should be the case for the lattices. | |
1050 | This should result in huge pages except THP is disabled on the machine. | |
1051 | (NOTE: madvise() is used if <tt class="docutils literal">HAVE_HUGE_PAGES</tt> is defined, which is currently | |
1052 | hard coded defined in <tt class="docutils literal">Memory.c</tt>).</li> | |
ecf590ae MW |
1053 | <li>CPU/core frequency: For reproducible results the frequency of all cores |
1054 | should be fixed.</li> | |
1055 | <li>NUMA placement policy: The benchmark assumes a first touch policy, which | |
1056 | means the memory will be placed at the NUMA domain the touching core is | |
1057 | associated with. If a different policy is in place or the NUMA domain to be | |
1058 | used is already full memory might be allocated in a remote domain. Accesses | |
1059 | to remote domains typically have a higher latency and lower bandwidth.</li> | |
0095f461 | 1060 | <li>System load: interference with other application, especially on desktop |
ecf590ae | 1061 | systems should be avoided.</li> |
e3f82424 MW |
1062 | <li>Padding: For SoA based kernels the number of (fluid) nodes is automatically |
1063 | adjusted so that no cache or TLB thrashing should occur. The parameters are | |
1064 | optimized for current Intel based systems. For more details look into the | |
1065 | padding section.</li> | |
ecf590ae MW |
1066 | <li>CPU dispatcher function: the compiler might add different versions of a |
1067 | function for different ISA extensions. Make sure the code you might think is | |
1068 | executed is actually the code which is executed.</li> | |
1069 | </ul> | |
0095f461 | 1070 | </div> |
e3f82424 | 1071 | <div class="section" id="padding"> |
0fde6e45 | 1072 | <h2><a class="toc-backref" href="#id21">4.4 Padding</a></h2> |
e3f82424 MW |
1073 | <p>With correct padding cache and TLB thrashing can be avoided. Therefore the |
1074 | number of (fluid) nodes used in the data layout is artificially increased.</p> | |
1075 | <p>Currently automatic padding is active for kernels which support it. It can be | |
1076 | controlled via the kernel parameter (i.e. parameter after the <tt class="docutils literal"><span class="pre">--</span></tt>) | |
1077 | <tt class="docutils literal"><span class="pre">-pad</span></tt>. Supported values are <tt class="docutils literal">auto</tt> (default), <tt class="docutils literal">no</tt> (to disable padding), | |
1078 | or a manual padding.</p> | |
1079 | <p>Automatic padding tries to avoid cache and TLB thrashing and pads for a 32 | |
1080 | entry (huge pages) TLB with 8 sets and a 512 set (L2) cache. This reflects the | |
1081 | parameters of current Intel based processors.</p> | |
1082 | <p>Manual padding is done via a padding string and has the format | |
1083 | <tt class="docutils literal"><span class="pre">mod_1+offset_1(,mod_n+offset_n)</span></tt>, which specifies numbers of bytes. | |
1084 | SoA data layouts can exhibit TLB thrashing. Therefore we want to distribute the | |
1085 | 19 pages with one lattice (36 with two lattices) we are concurrently accessing | |
1086 | over as much sets in the TLB as possible. | |
1087 | This is controlled by the distance between the accessed pages, which is the | |
1088 | number of (fluid) nodes in between them and can be adjusted by adding further | |
1089 | (fluid) nodes. | |
1090 | We want the distance d (in bytes) between two accessed pages to be e.g. | |
1091 | <strong>d % (PAGE_SIZE * TLB_SETS) = PAGE_SIZE</strong>. | |
1092 | This would distribute the pages evenly over the sets. Hereby <strong>PAGE_SIZE * TLB_SETS</strong> | |
1093 | would be our <tt class="docutils literal">mod_1</tt> and <strong>PAGE_SIZE</strong> (after the =) our <tt class="docutils literal">offset_1</tt>. | |
1094 | Measurements show that with only a quarter of half of a page size as offset | |
1095 | higher performance is achieved, which is done by automatic padding. | |
1096 | On top of this padding more paddings can be added. They are just added to the | |
1097 | padding string and are separated by commas.</p> | |
1098 | <p>A zero modulus in the padding string has a special meaning. Here the | |
1099 | corresponding offset is just added to the number of nodes. A padding string | |
1100 | like <tt class="docutils literal"><span class="pre">-pad</span> 0+16</tt> would at a static padding of two nodes (one node = 8 b).</p> | |
1101 | </div> | |
1102 | </div> | |
1103 | <div class="section" id="geometries"> | |
0fde6e45 | 1104 | <h1><a class="toc-backref" href="#id22">5 Geometries</a></h1> |
0095f461 MW |
1105 | <p>TODO: supported geometries: channel, pipe, blocks, fluid</p> |
1106 | </div> | |
1107 | <div class="section" id="performance-results"> | |
0fde6e45 | 1108 | <h1><a class="toc-backref" href="#id23">6 Performance Results</a></h1> |
0095f461 | 1109 | <p>The sections lists performance values measured on several machines for |
0fde6e45 | 1110 | different kernels and geometries and <strong>double precision</strong> floating point data/arithmetic. |
0095f461 MW |
1111 | The <strong>RFM</strong> column denotes the expected performance as predicted by the |
1112 | Roofline performance model <a class="citation-reference" href="#williams-2008" id="id3">[williams-2008]</a>. | |
1113 | For performance prediction of each kernel a memory bandwidth benchmark is used | |
1114 | which mimics the kernels memory access pattern and the kernel's loop balance | |
1115 | (see <a class="citation-reference" href="#kernels" id="id4">[kernels]</a> for details).</p> | |
0fde6e45 MW |
1116 | <div class="section" id="machine-specifications"> |
1117 | <h2><a class="toc-backref" href="#id24">6.1 Machine Specifications</a></h2> | |
1118 | <p><strong>Ivy Bridge, Intel Xeon E5-2660 v2</strong></p> | |
0095f461 | 1119 | <ul class="simple"> |
0fde6e45 MW |
1120 | <li>Ivy Bridge architecture, AVX</li> |
1121 | <li>10 cores, 2.2 GHz</li> | |
0095f461 | 1122 | <li>SMT enabled</li> |
0fde6e45 MW |
1123 | <li>memoy bandwidth:<ul> |
1124 | <li>copy-19 32.7 GB/s</li> | |
1125 | <li>copy-19-nt-sl 35.6 GB/s</li> | |
1126 | <li>update-19 37.4 GB/s</li> | |
1127 | </ul> | |
1128 | </li> | |
0095f461 | 1129 | </ul> |
0fde6e45 | 1130 | <p><strong>Haswell, Intel Xeon E5-2695 v3</strong></p> |
0095f461 | 1131 | <ul class="simple"> |
0fde6e45 MW |
1132 | <li>Haswell architecture, AVX2, FMA</li> |
1133 | <li>14 cores, 2.3 GHz</li> | |
1134 | <li>2 x 7 cores in cluster-on-die (CoD) mode enabled</li> | |
1135 | <li>SMT enabled</li> | |
1136 | <li>memory bandwidth:<ul> | |
0095f461 MW |
1137 | <li>copy-19 47.3 GB/s</li> |
1138 | <li>copy-19-nt-sl 47.1 GB/s</li> | |
1139 | <li>update-19 44.0 GB/s</li> | |
1140 | </ul> | |
0fde6e45 MW |
1141 | </li> |
1142 | </ul> | |
1143 | <p><strong>Broadwell, Intel Xeon E5-2630 v4</strong></p> | |
0095f461 MW |
1144 | <ul class="simple"> |
1145 | <li>Broadwell architecture, AVX2, FMA</li> | |
1146 | <li>10 cores, 2.2 GHz</li> | |
1147 | <li>SMT disabled</li> | |
0fde6e45 | 1148 | <li>memory bandwidth:<ul> |
0095f461 MW |
1149 | <li>copy-19 48.0 GB/s</li> |
1150 | <li>copy-nt-sl-19 48.2 GB/s</li> | |
1151 | <li>update-19 51.1 GB/s</li> | |
1152 | </ul> | |
0fde6e45 MW |
1153 | </li> |
1154 | </ul> | |
1155 | <p><strong>Skylake, Intel Xeon Gold 6148</strong></p> | |
1156 | <p>NOTE: currently we only use AVX2 intrinsics.</p> | |
1157 | <ul class="simple"> | |
1158 | <li>Skylake server architecture, AVX2, AVX512, 2 FMA units</li> | |
1159 | <li>20 cores, 2.4 GHz</li> | |
1160 | <li>SMT enabled</li> | |
1161 | <li>memory bandwidth:<ul> | |
1162 | <li>copy-19 89.7 GB/s</li> | |
1163 | <li>copy-19-nt-sl 92.4 GB/s</li> | |
1164 | <li>update-19 93.6 GB/s</li> | |
1165 | </ul> | |
1166 | </li> | |
1167 | </ul> | |
1168 | <p><strong>Zen, AMD EPYC 7451</strong></p> | |
1169 | <ul class="simple"> | |
1170 | <li>Zen architecture, AVX2, FMA</li> | |
1171 | <li>24 cores, 2.3 GHz</li> | |
1172 | <li>SMT enabled</li> | |
1173 | <li>memory bandwidth:<ul> | |
1174 | <li>copy-19 111.9 GB/s</li> | |
1175 | <li>copy-19-nt-sl 111.7 GB/s</li> | |
1176 | <li>update-19 109.2 GB/s</li> | |
1177 | </ul> | |
1178 | </li> | |
1179 | </ul> | |
1180 | <p><strong>Zen, AMD Ryzen 7 1700X</strong></p> | |
1181 | <ul class="simple"> | |
1182 | <li>Zen architecture, AVX2, FMA</li> | |
1183 | <li>8 cores, 3.4 GHz</li> | |
1184 | <li>SMT enabled</li> | |
1185 | <li>memory bandwidth:<ul> | |
1186 | <li>copy-19 27.2 GB/s</li> | |
1187 | <li>copy-19-nt-sl 27.1 GB/s</li> | |
1188 | <li>update-19 26.1 GB/s</li> | |
1189 | </ul> | |
1190 | </li> | |
1191 | </ul> | |
1192 | </div> | |
1193 | <div class="section" id="single-socket-results"> | |
1194 | <h2><a class="toc-backref" href="#id25">6.2 Single Socket Results</a></h2> | |
1195 | <ul class="simple"> | |
1196 | <li>Geometry dimensions are for all measurements 500x100x100 nodes.</li> | |
1197 | <li>Note the <strong>different scaling on the y axis</strong> of the plots!</li> | |
1198 | </ul> | |
0095f461 MW |
1199 | <table border="1" class="docutils"> |
1200 | <colgroup> | |
0fde6e45 | 1201 | <col width="100%" /> |
0095f461 | 1202 | </colgroup> |
0095f461 | 1203 | <tbody valign="top"> |
0fde6e45 | 1204 | <tr><td>Ivy Bridge, Intel Xeon E5-2660 v2, Double Precision</td> |
0095f461 | 1205 | </tr> |
0fde6e45 | 1206 | <tr><td><img alt="perf_emmy_dp" src="images/benchmark-emmy-dp.png" style="width: 1000.0px; height: 250.0px;" /></td> |
0095f461 | 1207 | </tr> |
0fde6e45 | 1208 | <tr><td>Ivy Bridge, Intel Xeon E5-2660 v2, Single Precision</td> |
0095f461 | 1209 | </tr> |
0fde6e45 | 1210 | <tr><td><img alt="perf_emmy_sp" src="images/benchmark-emmy-sp.png" style="width: 1000.0px; height: 250.0px;" /></td> |
0095f461 | 1211 | </tr> |
0fde6e45 | 1212 | <tr><td>Haswell, Intel Xeon E5-2695 v3, Double Precision</td> |
0095f461 | 1213 | </tr> |
0fde6e45 | 1214 | <tr><td><img alt="perf_hasep1_dp" src="images/benchmark-hasep1-dp.png" style="width: 1000.0px; height: 250.0px;" /></td> |
0095f461 | 1215 | </tr> |
0fde6e45 | 1216 | <tr><td>Haswell, Intel Xeon E5-2695 v3, Single Precision</td> |
0095f461 | 1217 | </tr> |
0fde6e45 | 1218 | <tr><td><img alt="perf_hasep1_sp" src="images/benchmark-hasep1-sp.png" style="width: 1000.0px; height: 250.0px;" /></td> |
0095f461 | 1219 | </tr> |
0fde6e45 | 1220 | <tr><td>Broadwell, Intel Xeon E5-2630 v4, Double Precision</td> |
0095f461 | 1221 | </tr> |
0fde6e45 | 1222 | <tr><td><img alt="perf_meggie_dp" src="images/benchmark-meggie-dp.png" style="width: 1000.0px; height: 250.0px;" /></td> |
0095f461 | 1223 | </tr> |
0fde6e45 | 1224 | <tr><td>Broadwell, Intel Xeon E5-2630 v4, Single Precision</td> |
0095f461 | 1225 | </tr> |
0fde6e45 | 1226 | <tr><td><img alt="perf_meggie_sp" src="images/benchmark-meggie-sp.png" style="width: 1000.0px; height: 250.0px;" /></td> |
0095f461 | 1227 | </tr> |
0fde6e45 | 1228 | <tr><td>Skylake, Intel Xeon Gold 6148, Double Precision, <strong>NOTE: currently we only use AVX2 intrinsics.</strong></td> |
0095f461 | 1229 | </tr> |
0fde6e45 | 1230 | <tr><td><img alt="perf_skylakesp2_dp" src="images/benchmark-skylakesp2-dp.png" style="width: 1000.0px; height: 250.0px;" /></td> |
0095f461 | 1231 | </tr> |
0fde6e45 | 1232 | <tr><td>Skylake, Intel Xeon Gold 6148, Single Precision, <strong>NOTE: currently we only use AVX2 intrinsics.</strong></td> |
0095f461 | 1233 | </tr> |
0fde6e45 | 1234 | <tr><td><img alt="perf_skylakesp2_sp" src="images/benchmark-skylakesp2-sp.png" style="width: 1000.0px; height: 250.0px;" /></td> |
0095f461 | 1235 | </tr> |
0fde6e45 | 1236 | <tr><td>Zen, AMD Ryzen 7 1700X, Double Precision</td> |
0095f461 | 1237 | </tr> |
0fde6e45 | 1238 | <tr><td><img alt="perf_summitridge1_dp" src="images/benchmark-summitridge1-dp.png" style="width: 1000.0px; height: 250.0px;" /></td> |
0095f461 | 1239 | </tr> |
0fde6e45 | 1240 | <tr><td>Zen, AMD Ryzen 7 1700X, Single Precision</td> |
0095f461 | 1241 | </tr> |
0fde6e45 | 1242 | <tr><td><img alt="perf_summitridge1_sp" src="images/benchmark-summitridge1-sp.png" style="width: 1000.0px; height: 250.0px;" /></td> |
0095f461 | 1243 | </tr> |
0fde6e45 | 1244 | <tr><td>Zen, AMD EPYC 7451, Double Precision</td> |
0095f461 | 1245 | </tr> |
0fde6e45 | 1246 | <tr><td><img alt="perf_naples1_dp" src="images/benchmark-naples1-dp.png" style="width: 1000.0px; height: 250.0px;" /></td> |
0095f461 | 1247 | </tr> |
0fde6e45 | 1248 | <tr><td>Zen, AMD EPYC 7451, Single Precision</td> |
0095f461 | 1249 | </tr> |
0fde6e45 | 1250 | <tr><td><img alt="perf_naples1_sp" src="images/benchmark-naples1-sp.png" style="width: 1000.0px; height: 250.0px;" /></td> |
0095f461 MW |
1251 | </tr> |
1252 | </tbody> | |
1253 | </table> | |
e3f82424 | 1254 | </div> |
e3f82424 MW |
1255 | </div> |
1256 | <div class="section" id="licence"> | |
0095f461 | 1257 | <h1><a class="toc-backref" href="#id26">7 Licence</a></h1> |
e3f82424 | 1258 | <p>The Lattice Boltzmann Benchmark Kernels are licensed under GPLv3.</p> |
ecf590ae MW |
1259 | </div> |
1260 | <div class="section" id="acknowledgements"> | |
0095f461 | 1261 | <h1><a class="toc-backref" href="#id27">8 Acknowledgements</a></h1> |
ecf590ae MW |
1262 | <p>This work was funded by BMBF, grant no. 01IH15003A (project SKAMPY).</p> |
1263 | <p>This work was funded by KONWHIR project OMI4PAPS.</p> | |
0095f461 MW |
1264 | </div> |
1265 | <div class="section" id="bibliography"> | |
1266 | <h1><a class="toc-backref" href="#id28">9 Bibliography</a></h1> | |
1267 | <table class="docutils citation" frame="void" id="ginzburg-2008" rules="none"> | |
1268 | <colgroup><col class="label" /><col /></colgroup> | |
1269 | <tbody valign="top"> | |
1270 | <tr><td class="label"><a class="fn-backref" href="#id1">[ginzburg-2008]</a></td><td>I. Ginzburg, F. Verhaeghe, and D. d'Humières. | |
1271 | Two-relaxation-time lattice Boltzmann scheme: About parametrization, velocity, pressure and mixed boundary conditions. | |
1272 | Commun. Comput. Phys., 3(2):427-478, 2008.</td></tr> | |
1273 | </tbody> | |
1274 | </table> | |
1275 | <table class="docutils citation" frame="void" id="williams-2008" rules="none"> | |
1276 | <colgroup><col class="label" /><col /></colgroup> | |
1277 | <tbody valign="top"> | |
1278 | <tr><td class="label"><a class="fn-backref" href="#id3">[williams-2008]</a></td><td>S. Williams, A. Waterman, and D. Patterson. | |
1279 | Roofline: an insightful visual performance model for multicore architectures. | |
1280 | Commun. ACM, 52(4):65-76, Apr 2009. doi:10.1145/1498765.1498785</td></tr> | |
1281 | </tbody> | |
1282 | </table> | |
8cafd9ea | 1283 | <p>Document was generated at 2018-05-10 14:10.</p> |
ecf590ae MW |
1284 | </div> |
1285 | </div> | |
1286 | </body> | |
1287 | </html> |