src/BenchKernelD3Q19.c

   1 // --------------------------------------------------------------------------
   2 //
   3 // Copyright
   4 //   Markus Wittmann, 2016-2017
   5 //   RRZE, University of Erlangen-Nuremberg, Germany
   6 //   markus.wittmann -at- fau.de or hpc -at- rrze.fau.de
   7 //
   8 //   Viktor Haag, 2016
   9 //   LSS, University of Erlangen-Nuremberg, Germany
  10 //
  11 //  This file is part of the Lattice Boltzmann Benchmark Kernels (LbmBenchKernels).
  12 //
  13 //  LbmBenchKernels is free software: you can redistribute it and/or modify
  14 //  it under the terms of the GNU General Public License as published by
  15 //  the Free Software Foundation, either version 3 of the License, or
  16 //  (at your option) any later version.
  17 //
  18 //  LbmBenchKernels is distributed in the hope that it will be useful,
  19 //  but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 //  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 //  GNU General Public License for more details.
  22 //
  23 //  You should have received a copy of the GNU General Public License
  24 //  along with LbmBenchKernels.  If not, see <http://www.gnu.org/licenses/>.
  25 //
  26 // --------------------------------------------------------------------------
  27 #include "BenchKernelD3Q19Common.h"
  28
  29 #include "Memory.h"
  30 #include "Vtk.h"
  31 #include "LikwidIf.h"
  32
  33 #include <inttypes.h>
  34 #include <math.h>
  35
  36 #ifdef _OPENMP
  37         #include <omp.h>
  38 #endif
  39
  40 void FNAME(D3Q19Kernel)(LatticeDesc * ld, KernelData * kernelData, CaseData * cd)
  41 {
  42         Assert(ld != NULL);
  43         Assert(kernelData != NULL);
  44         Assert(cd != NULL);
  45
  46         Assert(cd->Omega > 0.0);
  47         Assert(cd->Omega < 2.0);
  48
  49         KernelData * kd = (KernelData *)kernelData;
  50
  51
  52         int nX = ld->Dims[0];
  53         int nY = ld->Dims[1];
  54         int nZ = ld->Dims[2];
  55
  56         int * gDims = kd->GlobalDims;
  57
  58         int oX = kd->Offsets[0];
  59         int oY = kd->Offsets[1];
  60         int oZ = kd->Offsets[2];
  61
  62         PdfT omega = cd->Omega;
  63         PdfT omegaEven = omega;
  64 //      PdfT omegaOdd = 8.0*((2.0-omegaEven)/(8.0-omegaEven)); //"standard" trt odd relaxation parameter
  65         PdfT magicParam = 1.0/12.0;     // 1/4: best stability; 1/12: removes third-order advection error (best advection);     1/6: removes fourth-order diffusion error (best diffusion);     3/16: exact location of bounce back for poiseuille flow
  66         PdfT omegaOdd = 1.0/( 0.5 + magicParam/(1.0/omega - 0.5) );
  67
  68         PdfT evenPart = 0.0;
  69         PdfT oddPart = 0.0;
  70         PdfT dir_indep_trm = 0.0;
  71
  72         PdfT w_0 = 1.0 /  3.0;
  73         PdfT w_1 = 1.0 / 18.0;
  74         PdfT w_2 = 1.0 / 36.0;
  75
  76         PdfT w_1_x3 = w_1 * 3.0;        PdfT w_1_nine_half = w_1 * 9.0/2.0;     PdfT w_1_indep = 0.0;
  77         PdfT w_2_x3 = w_2 * 3.0;        PdfT w_2_nine_half = w_2 * 9.0/2.0;     PdfT w_2_indep = 0.0;
  78
  79         PdfT ux, uy, uz, ui;
  80         PdfT dens;
  81
  82         // Declare pdf_N, pdf_E, pdf_S, pdf_W, ...
  83         #define X(name, idx, idxinv, x, y, z)   PdfT JOIN(pdf_,name);
  84         D3Q19_LIST
  85         #undef X
  86
  87         PdfT * src = kd->Pdfs[0];
  88         PdfT * dst = kd->Pdfs[1];
  89         PdfT * tmp;
  90
  91         int maxIterations = cd->MaxIterations;
  92
  93         #ifdef VTK_OUTPUT
  94         if (cd->VtkOutput) {
  95                 kd->PdfsActive = src;
  96                 VtkWrite(ld, kd, cd, 0);
  97         }
  98         #endif
  99
 100         for (int iter = 0; iter < maxIterations; ++iter) {
 101
 102                 X_LIKWID_START("os");
 103
 104                 #ifdef _OPENMP
 105                 #pragma omp parallel for collapse(3) default(none) \
 106                                 shared(gDims,src, dst, w_0, w_1, w_2, omegaEven, omegaOdd, \
 107                                 w_1_x3, w_2_x3, w_1_nine_half, w_2_nine_half, cd, \
 108                                 oX, oY, oZ, nX, nY, nZ) \
 109                                 private(ux, uy, uz, ui, dens, dir_indep_trm, \
 110                                         pdf_C, \
 111                                         pdf_N, pdf_E, pdf_S, pdf_W, \
 112                                         pdf_NE, pdf_SE, pdf_SW, pdf_NW, \
 113                                         pdf_T, pdf_TN, pdf_TE, pdf_TS, pdf_TW, \
 114                                         pdf_B, pdf_BN, pdf_BE, pdf_BS, pdf_BW, \
 115                                         evenPart, oddPart, w_1_indep, w_2_indep)
 116                 #endif
 117
 118                 for (int x = oX; x < nX + oX; ++x) {
 119                         for (int y = oY; y < nY + oY; ++y) {
 120                                 for (int z = oZ; z < nZ + oZ; ++z) {
 121                                         #define I(x, y, z, dir) P_INDEX_5(gDims, (x), (y), (z), (dir))
 122
 123 #ifdef PROP_MODEL_PUSH
 124
 125                                         // Load PDFs of local cell: pdf_N = src[I(x, y, z, D3Q19_N)]; ...
 126                                         #define X(name, idx, idxinv, _x, _y, _z)        JOIN(pdf_,name) = src[I(x, y, z, idx)];
 127                                         //if (isnan(JOIN(pdf_,name))) { printf("iter: %d %d %d %d %d %s nan\n", iter, x-oX, y-oY, z-oZ, idx, D3Q19_NAMES[idx]); exit(1);}
 128                                         D3Q19_LIST
 129                                         #undef X
 130
 131 #elif PROP_MODEL_PULL
 132
 133                                         // Load PDFs of local cell: pdf_N = src[I(x, y, z, D3Q19_N)]; ...
 134                                         #define X(name, idx, idxinv, _x, _y, _z)        JOIN(pdf_,name) = src[I(x - _x, y - _y, z - _z, idx)];
 135                                         //if (isnan(JOIN(pdf_,name))) { printf("iter: %d %d %d %d %d %s nan\n", iter, x-oX, y-oY, z-oZ, idx, D3Q19_NAMES[idx]); exit(1);}
 136                                         D3Q19_LIST
 137                                         #undef X
 138
 139 #else
 140         #error No implementation for PROP_MODEL_NAME.
 141 #endif
 142
 143                                         // #define LID_DRIVEN_CAVITY
 144
 145                                         #ifdef LID_DRIVEN_CAVITY
 146
 147                                         if (z == nZ - 4 + oZ && x > 3 + oX && x < (nX - 4 + oX) && y > 3 + oY  && y < (nY - 4 + oY)) {
 148                                                 ux = 0.1 * 0.577;
 149                                                 uy = 0.0;
 150                                                 uz = 0.0;
 151
 152                                         } else {
 153                                         #endif
 154                                                 ux = pdf_E + pdf_NE + pdf_SE + pdf_TE + pdf_BE -
 155                                                         pdf_W - pdf_NW - pdf_SW - pdf_TW - pdf_BW;
 156                                                 uy = pdf_N + pdf_NE + pdf_NW + pdf_TN + pdf_BN -
 157                                                         pdf_S - pdf_SE - pdf_SW - pdf_TS - pdf_BS;
 158                                                 uz = pdf_T + pdf_TE + pdf_TW + pdf_TN + pdf_TS -
 159                                                         pdf_B - pdf_BE - pdf_BW - pdf_BN - pdf_BS;
 160                                         #ifdef LID_DRIVEN_CAVITY
 161                                         }
 162
 163                                         #endif
 164
 165                                         dens = pdf_C +
 166                                                         pdf_N  + pdf_E  + pdf_S  + pdf_W  +
 167                                                         pdf_NE + pdf_SE + pdf_SW + pdf_NW +
 168                                                         pdf_T  + pdf_TN + pdf_TE + pdf_TS + pdf_TW +
 169                                                         pdf_B  + pdf_BN + pdf_BE + pdf_BS + pdf_BW;
 170
 171                                         dir_indep_trm = dens - (ux * ux + uy * uy + uz * uz)*3.0/2.0;
 172
 173 #ifdef PROP_MODEL_PUSH
 174
 175                                         // direction: w_0
 176                                         dst[I(x,   y,   z, D3Q19_C)]  = pdf_C - omegaEven*(pdf_C - w_0*dir_indep_trm);
 177
 178                                         // direction: w_1
 179                                         w_1_indep = w_1*dir_indep_trm;
 180
 181                                         ui = uy;
 182                                         evenPart = omegaEven*( 0.5*(pdf_N + pdf_S) - ui*ui*w_1_nine_half - w_1_indep );
 183                                         oddPart = omegaOdd*(0.5*(pdf_N - pdf_S) - ui*w_1_x3 );
 184                                         dst[I(x, y + 1,   z, D3Q19_N)]  = pdf_N - evenPart - oddPart;
 185                                         dst[I(x, y - 1,   z, D3Q19_S)]  = pdf_S - evenPart + oddPart;
 186
 187                                         ui = ux;
 188                                         evenPart = omegaEven*( 0.5*(pdf_E + pdf_W) - ui*ui*w_1_nine_half - w_1_indep );
 189                                         oddPart = omegaOdd*(0.5*(pdf_E - pdf_W) - ui*w_1_x3 );
 190                                         dst[I(x + 1,   y,   z, D3Q19_E)]  = pdf_E - evenPart - oddPart;
 191                                         dst[I(x - 1,   y,   z, D3Q19_W)]  = pdf_W - evenPart + oddPart;
 192
 193                                         ui = uz;
 194                                         evenPart = omegaEven*( 0.5*(pdf_T + pdf_B) - ui*ui*w_1_nine_half - w_1_indep );
 195                                         oddPart = omegaOdd*(0.5*(pdf_T - pdf_B) - ui*w_1_x3 );
 196                                         dst[I(x,   y, z + 1, D3Q19_T)]  = pdf_T - evenPart - oddPart;
 197                                         dst[I(x,   y, z - 1, D3Q19_B)]  = pdf_B - evenPart + oddPart;
 198
 199                                         // direction: w_2
 200                                         w_2_indep = w_2*dir_indep_trm;
 201
 202                                         ui = -ux + uy;
 203                                         evenPart = omegaEven*( 0.5*(pdf_NW + pdf_SE) - ui*ui*w_2_nine_half - w_2_indep );
 204                                         oddPart = omegaOdd*(0.5*(pdf_NW - pdf_SE) - ui*w_2_x3 );
 205                                         dst[I(x - 1, y + 1,   z, D3Q19_NW)] = pdf_NW - evenPart - oddPart;
 206                                         dst[I(x + 1, y - 1,   z, D3Q19_SE)] = pdf_SE - evenPart + oddPart;
 207
 208                                         ui = ux + uy;
 209                                         evenPart = omegaEven*( 0.5*(pdf_NE + pdf_SW) - ui*ui*w_2_nine_half - w_2_indep );
 210                                         oddPart = omegaOdd*(0.5*(pdf_NE - pdf_SW) - ui*w_2_x3 );
 211                                         dst[I(x + 1, y + 1,   z, D3Q19_NE)] = pdf_NE - evenPart - oddPart;
 212                                         dst[I(x - 1, y - 1,   z, D3Q19_SW)] = pdf_SW - evenPart + oddPart;
 213
 214                                         ui = -ux + uz;
 215                                         evenPart = omegaEven*( 0.5*(pdf_TW + pdf_BE) - ui*ui*w_2_nine_half - w_2_indep );
 216                                         oddPart = omegaOdd*(0.5*(pdf_TW - pdf_BE) - ui*w_2_x3 );
 217                                         dst[I(x - 1,   y, z + 1, D3Q19_TW)] = pdf_TW - evenPart - oddPart;
 218                                         dst[I(x + 1,   y, z - 1, D3Q19_BE)] = pdf_BE - evenPart + oddPart;
 219
 220                                         ui = ux + uz;
 221                                         evenPart = omegaEven*( 0.5*(pdf_TE + pdf_BW) - ui*ui*w_2_nine_half - w_2_indep );
 222                                         oddPart = omegaOdd*(0.5*(pdf_TE - pdf_BW) - ui*w_2_x3 );
 223                                         dst[I(x + 1,   y, z + 1, D3Q19_TE)] = pdf_TE - evenPart - oddPart;
 224                                         dst[I(x - 1,   y, z - 1, D3Q19_BW)] = pdf_BW - evenPart + oddPart;
 225
 226                                         ui = -uy + uz;
 227                                         evenPart = omegaEven*( 0.5*(pdf_TS + pdf_BN) - ui*ui*w_2_nine_half - w_2_indep );
 228                                         oddPart = omegaOdd*(0.5*(pdf_TS - pdf_BN) - ui*w_2_x3 );
 229                                         dst[I(x, y - 1, z + 1, D3Q19_TS)] = pdf_TS - evenPart - oddPart;
 230                                         dst[I(x, y + 1, z - 1, D3Q19_BN)] = pdf_BN - evenPart + oddPart;
 231
 232                                         ui = uy + uz;
 233                                         evenPart = omegaEven*( 0.5*(pdf_TN + pdf_BS) - ui*ui*w_2_nine_half - w_2_indep );
 234                                         oddPart = omegaOdd*(0.5*(pdf_TN - pdf_BS) - ui*w_2_x3 );
 235                                         dst[I(x, y + 1, z + 1, D3Q19_TN)] = pdf_TN - evenPart - oddPart;
 236                                         dst[I(x, y - 1, z - 1, D3Q19_BS)] = pdf_BS - evenPart + oddPart;
 237
 238 #elif PROP_MODEL_PULL
 239
 240                                         // direction: w_0
 241                                         dst[I(x,   y,   z, D3Q19_C)]  = pdf_C - omegaEven*(pdf_C - w_0*dir_indep_trm);
 242
 243                                         // direction: w_1
 244                                         w_1_indep = w_1*dir_indep_trm;
 245
 246                                         ui = uy;
 247                                         evenPart = omegaEven*( 0.5*(pdf_N + pdf_S) - ui*ui*w_1_nine_half - w_1_indep );
 248                                         oddPart = omegaOdd*(0.5*(pdf_N - pdf_S) - ui*w_1_x3 );
 249                                         dst[I(x, y, z, D3Q19_N)]  = pdf_N - evenPart - oddPart;
 250                                         dst[I(x, y, z, D3Q19_S)]  = pdf_S - evenPart + oddPart;
 251
 252                                         ui = ux;
 253                                         evenPart = omegaEven*( 0.5*(pdf_E + pdf_W) - ui*ui*w_1_nine_half - w_1_indep );
 254                                         oddPart = omegaOdd*(0.5*(pdf_E - pdf_W) - ui*w_1_x3 );
 255                                         dst[I(x, y, z, D3Q19_E)]  = pdf_E - evenPart - oddPart;
 256                                         dst[I(x, y, z, D3Q19_W)]  = pdf_W - evenPart + oddPart;
 257
 258                                         ui = uz;
 259                                         evenPart = omegaEven*( 0.5*(pdf_T + pdf_B) - ui*ui*w_1_nine_half - w_1_indep );
 260                                         oddPart = omegaOdd*(0.5*(pdf_T - pdf_B) - ui*w_1_x3 );
 261                                         dst[I(x, y, z, D3Q19_T)]  = pdf_T - evenPart - oddPart;
 262                                         dst[I(x, y, z, D3Q19_B)]  = pdf_B - evenPart + oddPart;
 263
 264                                         // direction: w_2
 265                                         w_2_indep = w_2*dir_indep_trm;
 266
 267                                         ui = -ux + uy;
 268                                         evenPart = omegaEven*( 0.5*(pdf_NW + pdf_SE) - ui*ui*w_2_nine_half - w_2_indep );
 269                                         oddPart = omegaOdd*(0.5*(pdf_NW - pdf_SE) - ui*w_2_x3 );
 270                                         dst[I(x, y, z, D3Q19_NW)] = pdf_NW - evenPart - oddPart;
 271                                         dst[I(x, y, z, D3Q19_SE)] = pdf_SE - evenPart + oddPart;
 272
 273                                         ui = ux + uy;
 274                                         evenPart = omegaEven*( 0.5*(pdf_NE + pdf_SW) - ui*ui*w_2_nine_half - w_2_indep );
 275                                         oddPart = omegaOdd*(0.5*(pdf_NE - pdf_SW) - ui*w_2_x3 );
 276                                         dst[I(x, y, z, D3Q19_NE)] = pdf_NE - evenPart - oddPart;
 277                                         dst[I(x, y, z, D3Q19_SW)] = pdf_SW - evenPart + oddPart;
 278
 279                                         ui = -ux + uz;
 280                                         evenPart = omegaEven*( 0.5*(pdf_TW + pdf_BE) - ui*ui*w_2_nine_half - w_2_indep );
 281                                         oddPart = omegaOdd*(0.5*(pdf_TW - pdf_BE) - ui*w_2_x3 );
 282                                         dst[I(x, y, z, D3Q19_TW)] = pdf_TW - evenPart - oddPart;
 283                                         dst[I(x, y, z, D3Q19_BE)] = pdf_BE - evenPart + oddPart;
 284
 285                                         ui = ux + uz;
 286                                         evenPart = omegaEven*( 0.5*(pdf_TE + pdf_BW) - ui*ui*w_2_nine_half - w_2_indep );
 287                                         oddPart = omegaOdd*(0.5*(pdf_TE - pdf_BW) - ui*w_2_x3 );
 288                                         dst[I(x, y, z, D3Q19_TE)] = pdf_TE - evenPart - oddPart;
 289                                         dst[I(x, y, z, D3Q19_BW)] = pdf_BW - evenPart + oddPart;
 290
 291                                         ui = -uy + uz;
 292                                         evenPart = omegaEven*( 0.5*(pdf_TS + pdf_BN) - ui*ui*w_2_nine_half - w_2_indep );
 293                                         oddPart = omegaOdd*(0.5*(pdf_TS - pdf_BN) - ui*w_2_x3 );
 294                                         dst[I(x, y, z, D3Q19_TS)] = pdf_TS - evenPart - oddPart;
 295                                         dst[I(x, y, z, D3Q19_BN)] = pdf_BN - evenPart + oddPart;
 296
 297                                         ui = uy + uz;
 298                                         evenPart = omegaEven*( 0.5*(pdf_TN + pdf_BS) - ui*ui*w_2_nine_half - w_2_indep );
 299                                         oddPart = omegaOdd*(0.5*(pdf_TN - pdf_BS) - ui*w_2_x3 );
 300                                         dst[I(x, y, z, D3Q19_TN)] = pdf_TN - evenPart - oddPart;
 301                                         dst[I(x, y, z, D3Q19_BS)] = pdf_BS - evenPart + oddPart;
 302
 303 #else
 304         #error No implementation for PROP_MODEL_NAME.
 305 #endif
 306
 307                                         #undef I
 308                                 }
 309                         }
 310                 } // z, y, x (from inner to outer)
 311
 312                 // Stop counters before bounce back. Else computing loop balance will be incorrect.
 313                 X_LIKWID_STOP("os");
 314
 315                 // Fixup bounce back PDFs.
 316                 #ifdef _OPENMP
 317                 #pragma omp parallel for default(none) \
 318                                 shared(kd, dst)
 319                 #endif
 320                 for (int i = 0; i < kd->nBounceBackPdfs; ++i) {
 321                         dst[kd->BounceBackPdfsDst[i]] = dst[kd->BounceBackPdfsSrc[i]];
 322                 }
 323
 324                 #ifdef VERIFICATION
 325                 kd->PdfsActive = dst;
 326                 KernelAddBodyForce(kd, ld, cd);
 327                 #endif
 328
 329                 #ifdef VTK_OUTPUT
 330
 331                 if (cd->VtkOutput && (iter % cd->VtkModulus) == 0) {
 332                         kd->PdfsActive = dst;
 333                         VtkWrite(ld, kd, cd, iter);
 334                 }
 335
 336                 #endif
 337
 338                 #ifdef STATISTICS
 339                 kd->PdfsActive = dst;
 340                 KernelStatistics(kd, ld, cd, iter);
 341                 #endif
 342
 343                 // swap grids
 344                 tmp = src;
 345                 src = dst;
 346                 dst = tmp;
 347
 348         } // for (int iter = 0; ...
 349
 350         #ifdef VTK_OUTPUT
 351
 352         if (cd->VtkOutput) {
 353                 kd->PdfsActive = src;
 354                 VtkWrite(ld, kd, cd, maxIterations);
 355         }
 356
 357         #endif
 358
 359         return;
 360 }
 361
 362
 363 void FNAME(D3Q19BlkKernel)(LatticeDesc * ld, KernelData * kernelData, CaseData * cd)
 364 {
 365         Assert(ld != NULL);
 366         Assert(kernelData != NULL);
 367         Assert(cd != NULL);
 368
 369         Assert(cd->Omega > 0.0);
 370         Assert(cd->Omega < 2.0);
 371
 372         KernelData * kd = (KernelData *)kernelData;
 373
 374
 375         int nX = ld->Dims[0];
 376         int nY = ld->Dims[1];
 377         int nZ = ld->Dims[2];
 378
 379         int * gDims = kd->GlobalDims;
 380
 381         int oX = kd->Offsets[0];
 382         int oY = kd->Offsets[1];
 383         int oZ = kd->Offsets[2];
 384
 385         KernelDataEx * kdex = (KernelDataEx *)kd;
 386
 387         int blk[3];
 388         blk[0] = kdex->Blk[0];
 389         blk[1] = kdex->Blk[1];
 390         blk[2] = kdex->Blk[2];
 391
 392         PdfT omega = cd->Omega;
 393         PdfT omegaEven = omega;
 394 //      PdfT omegaOdd = 8.0*((2.0-omegaEven)/(8.0-omegaEven)); //"standard" trt odd relaxation parameter
 395         PdfT magicParam = 1.0/12.0;     // 1/4: best stability; 1/12: removes third-order advection error (best advection);     1/6: removes fourth-order diffusion error (best diffusion);     3/16: exact location of bounce back for poiseuille flow
 396         PdfT omegaOdd = 1.0/( 0.5 + magicParam/(1.0/omega - 0.5) );
 397
 398         PdfT evenPart = 0.0;
 399         PdfT oddPart = 0.0;
 400         PdfT dir_indep_trm = 0.0;
 401
 402         PdfT w_0 = 1.0 /  3.0;
 403         PdfT w_1 = 1.0 / 18.0;
 404         PdfT w_2 = 1.0 / 36.0;
 405
 406         PdfT w_1_x3 = w_1 * 3.0;        PdfT w_1_nine_half = w_1 * 9.0/2.0;     PdfT w_1_indep = 0.0;
 407         PdfT w_2_x3 = w_2 * 3.0;        PdfT w_2_nine_half = w_2 * 9.0/2.0;     PdfT w_2_indep = 0.0;
 408
 409         PdfT ux, uy, uz, ui;
 410         PdfT dens;
 411
 412         // Declare pdf_N, pdf_E, pdf_S, pdf_W, ...
 413         #define X(name, idx, idxinv, x, y, z)   PdfT JOIN(pdf_,name);
 414         D3Q19_LIST
 415         #undef X
 416
 417         PdfT * src = kd->Pdfs[0];
 418         PdfT * dst = kd->Pdfs[1];
 419         PdfT * tmp;
 420
 421         int maxIterations = cd->MaxIterations;
 422
 423         #ifdef VTK_OUTPUT
 424         if (cd->VtkOutput) {
 425                 kd->PdfsActive = src;
 426                 VtkWrite(ld, kd, cd, 0);
 427         }
 428         #endif
 429
 430         int nThreads = 1;
 431
 432         #ifdef _OPENMP
 433         nThreads = omp_get_max_threads();
 434         #endif
 435
 436         for (int iter = 0; iter < maxIterations; ++iter) {
 437
 438                 #ifdef _OPENMP
 439                 #pragma omp parallel default(none) \
 440                                 shared(gDims,src, dst, w_0, w_1, w_2, omegaEven, omegaOdd, \
 441                                 w_1_x3, w_2_x3, w_1_nine_half, w_2_nine_half, cd, \
 442                                 oX, oY, oZ, nX, nY, nZ, blk, nThreads) \
 443                                 private(ux, uy, uz, ui, dens, dir_indep_trm, \
 444                                         pdf_C, \
 445                                         pdf_N, pdf_E, pdf_S, pdf_W, \
 446                                         pdf_NE, pdf_SE, pdf_SW, pdf_NW, \
 447                                         pdf_T, pdf_TN, pdf_TE, pdf_TS, pdf_TW, \
 448                                         pdf_B, pdf_BN, pdf_BE, pdf_BS, pdf_BW, \
 449                                         evenPart, oddPart, w_1_indep, w_2_indep)
 450                 #endif
 451                 {
 452                         X_LIKWID_START("blk-os");
 453
 454                         int threadId = omp_get_thread_num();
 455
 456                         int threadStartX = nX / nThreads * threadId;
 457                         int threadEndX   = nX / nThreads * (threadId + 1);
 458
 459                         if (nX % nThreads > 0) {
 460                                 if (nX % nThreads > threadId) {
 461                                         threadStartX += threadId;
 462                                         threadEndX   += threadId + 1;
 463                                 }
 464                                 else {
 465                                         threadStartX += nX % nThreads;
 466                                         threadEndX   += nX % nThreads;
 467                                 }
 468                         }
 469
 470                 for (int bX = oX + threadStartX; bX < threadEndX + oX; bX += blk[0]) {
 471                         for (int bY = oY; bY < nY + oY; bY += blk[1]) {
 472                                 for (int bZ = oZ; bZ < nZ + oZ; bZ += blk[2]) {
 473
 474                                         // Must do everything here, else it would break collapse.
 475                                         int eZ = MIN(bZ + blk[2], nZ + oZ);
 476                                         int eY = MIN(bY + blk[1], nY + oY);
 477                                         int eX = MIN(bX + blk[0], threadEndX + oX);
 478
 479                                         for (int x = bX; x < eX; ++x) {
 480                                                 for (int y = bY; y < eY; ++y) {
 481                                                         for (int z = bZ; z < eZ; ++z) {
 482
 483                                         #define I(x, y, z, dir) P_INDEX_5(gDims, (x), (y), (z), (dir))
 484
 485 #ifdef PROP_MODEL_PUSH
 486
 487                                         // Load PDFs of local cell: pdf_N = src[I(x, y, z, D3Q19_N)]; ...
 488                                         #define X(name, idx, idxinv, _x, _y, _z)        JOIN(pdf_,name) = src[I(x, y, z, idx)];
 489                                         D3Q19_LIST
 490                                         #undef X
 491
 492 #elif PROP_MODEL_PULL
 493
 494                                         // Load PDFs of local cell: pdf_N = src[I(x, y, z, D3Q19_N)]; ...
 495                                         #define X(name, idx, idxinv, _x, _y, _z)        JOIN(pdf_,name) = src[I(x - _x, y - _y, z - _z, idx)];
 496                                         D3Q19_LIST
 497                                         #undef X
 498
 499 #else
 500         #error No implementation for PROP_MODEL_NAME.
 501 #endif
 502
 503                                         // #define LID_DRIVEN_CAVITY
 504
 505                                         #ifdef LID_DRIVEN_CAVITY
 506
 507                                         if (z == nZ - 4 + oZ && x > 3 + oX && x < (nX - 4 + oX) && y > 3 + oY  && y < (nY - 4 + oY)) {
 508                                                 ux = 0.1 * 0.577;
 509                                                 uy = 0.0;
 510                                                 uz = 0.0;
 511
 512                                         } else {
 513                                         #endif
 514                                                 ux = pdf_E + pdf_NE + pdf_SE + pdf_TE + pdf_BE -
 515                                                          pdf_W - pdf_NW - pdf_SW - pdf_TW - pdf_BW;
 516                                                 uy = pdf_N + pdf_NE + pdf_NW + pdf_TN + pdf_BN -
 517                                                          pdf_S - pdf_SE - pdf_SW - pdf_TS - pdf_BS;
 518                                                 uz = pdf_T + pdf_TE + pdf_TW + pdf_TN + pdf_TS -
 519                                                          pdf_B - pdf_BE - pdf_BW - pdf_BN - pdf_BS;
 520                                         #ifdef LID_DRIVEN_CAVITY
 521                                         }
 522
 523                                         #endif
 524
 525                                         dens = pdf_C +
 526                                                         pdf_N  + pdf_E  + pdf_S  + pdf_W  +
 527                                                         pdf_NE + pdf_SE + pdf_SW + pdf_NW +
 528                                                         pdf_T  + pdf_TN + pdf_TE + pdf_TS + pdf_TW +
 529                                                         pdf_B  + pdf_BN + pdf_BE + pdf_BS + pdf_BW;
 530
 531                                         dir_indep_trm = dens - (ux * ux + uy * uy + uz * uz)*3.0/2.0;
 532
 533 #ifdef PROP_MODEL_PUSH
 534
 535                                         // direction: w_0
 536                                         dst[I(x,   y,   z, D3Q19_C)]  = pdf_C - omegaEven*(pdf_C - w_0*dir_indep_trm);
 537
 538                                         // direction: w_1
 539                                         w_1_indep = w_1*dir_indep_trm;
 540
 541                                         ui = uy;
 542                                         evenPart = omegaEven*( 0.5*(pdf_N + pdf_S) - ui*ui*w_1_nine_half - w_1_indep );
 543                                         oddPart = omegaOdd*(0.5*(pdf_N - pdf_S) - ui*w_1_x3 );
 544                                         dst[I(x, y + 1,   z, D3Q19_N)]  = pdf_N - evenPart - oddPart;
 545                                         dst[I(x, y - 1,   z, D3Q19_S)]  = pdf_S - evenPart + oddPart;
 546
 547                                         ui = ux;
 548                                         evenPart = omegaEven*( 0.5*(pdf_E + pdf_W) - ui*ui*w_1_nine_half - w_1_indep );
 549                                         oddPart = omegaOdd*(0.5*(pdf_E - pdf_W) - ui*w_1_x3 );
 550                                         dst[I(x + 1,   y,   z, D3Q19_E)]  = pdf_E - evenPart - oddPart;
 551                                         dst[I(x - 1,   y,   z, D3Q19_W)]  = pdf_W - evenPart + oddPart;
 552
 553                                         ui = uz;
 554                                         evenPart = omegaEven*( 0.5*(pdf_T + pdf_B) - ui*ui*w_1_nine_half - w_1_indep );
 555                                         oddPart = omegaOdd*(0.5*(pdf_T - pdf_B) - ui*w_1_x3 );
 556                                         dst[I(x,   y, z + 1, D3Q19_T)]  = pdf_T - evenPart - oddPart;
 557                                         dst[I(x,   y, z - 1, D3Q19_B)]  = pdf_B - evenPart + oddPart;
 558
 559                                         // direction: w_2
 560                                         w_2_indep = w_2*dir_indep_trm;
 561
 562                                         ui = -ux + uy;
 563                                         evenPart = omegaEven*( 0.5*(pdf_NW + pdf_SE) - ui*ui*w_2_nine_half - w_2_indep );
 564                                         oddPart = omegaOdd*(0.5*(pdf_NW - pdf_SE) - ui*w_2_x3 );
 565                                         dst[I(x - 1, y + 1,   z, D3Q19_NW)] = pdf_NW - evenPart - oddPart;
 566                                         dst[I(x + 1, y - 1,   z, D3Q19_SE)] = pdf_SE - evenPart + oddPart;
 567
 568                                         ui = ux + uy;
 569                                         evenPart = omegaEven*( 0.5*(pdf_NE + pdf_SW) - ui*ui*w_2_nine_half - w_2_indep );
 570                                         oddPart = omegaOdd*(0.5*(pdf_NE - pdf_SW) - ui*w_2_x3 );
 571                                         dst[I(x + 1, y + 1,   z, D3Q19_NE)] = pdf_NE - evenPart - oddPart;
 572                                         dst[I(x - 1, y - 1,   z, D3Q19_SW)] = pdf_SW - evenPart + oddPart;
 573
 574                                         ui = -ux + uz;
 575                                         evenPart = omegaEven*( 0.5*(pdf_TW + pdf_BE) - ui*ui*w_2_nine_half - w_2_indep );
 576                                         oddPart = omegaOdd*(0.5*(pdf_TW - pdf_BE) - ui*w_2_x3 );
 577                                         dst[I(x - 1,   y, z + 1, D3Q19_TW)] = pdf_TW - evenPart - oddPart;
 578                                         dst[I(x + 1,   y, z - 1, D3Q19_BE)] = pdf_BE - evenPart + oddPart;
 579
 580                                         ui = ux + uz;
 581                                         evenPart = omegaEven*( 0.5*(pdf_TE + pdf_BW) - ui*ui*w_2_nine_half - w_2_indep );
 582                                         oddPart = omegaOdd*(0.5*(pdf_TE - pdf_BW) - ui*w_2_x3 );
 583                                         dst[I(x + 1,   y, z + 1, D3Q19_TE)] = pdf_TE - evenPart - oddPart;
 584                                         dst[I(x - 1,   y, z - 1, D3Q19_BW)] = pdf_BW - evenPart + oddPart;
 585
 586                                         ui = -uy + uz;
 587                                         evenPart = omegaEven*( 0.5*(pdf_TS + pdf_BN) - ui*ui*w_2_nine_half - w_2_indep );
 588                                         oddPart = omegaOdd*(0.5*(pdf_TS - pdf_BN) - ui*w_2_x3 );
 589                                         dst[I(x, y - 1, z + 1, D3Q19_TS)] = pdf_TS - evenPart - oddPart;
 590                                         dst[I(x, y + 1, z - 1, D3Q19_BN)] = pdf_BN - evenPart + oddPart;
 591
 592                                         ui = uy + uz;
 593                                         evenPart = omegaEven*( 0.5*(pdf_TN + pdf_BS) - ui*ui*w_2_nine_half - w_2_indep );
 594                                         oddPart = omegaOdd*(0.5*(pdf_TN - pdf_BS) - ui*w_2_x3 );
 595                                         dst[I(x, y + 1, z + 1, D3Q19_TN)] = pdf_TN - evenPart - oddPart;
 596                                         dst[I(x, y - 1, z - 1, D3Q19_BS)] = pdf_BS - evenPart + oddPart;
 597
 598 #elif PROP_MODEL_PULL
 599
 600                                         // direction: w_0
 601                                         dst[I(x,   y,   z, D3Q19_C)]  = pdf_C - omegaEven*(pdf_C - w_0*dir_indep_trm);
 602
 603                                         // direction: w_1
 604                                         w_1_indep = w_1*dir_indep_trm;
 605
 606                                         ui = uy;
 607                                         evenPart = omegaEven*( 0.5*(pdf_N + pdf_S) - ui*ui*w_1_nine_half - w_1_indep );
 608                                         oddPart = omegaOdd*(0.5*(pdf_N - pdf_S) - ui*w_1_x3 );
 609                                         dst[I(x, y, z, D3Q19_N)]  = pdf_N - evenPart - oddPart;
 610                                         dst[I(x, y, z, D3Q19_S)]  = pdf_S - evenPart + oddPart;
 611
 612                                         ui = ux;
 613                                         evenPart = omegaEven*( 0.5*(pdf_E + pdf_W) - ui*ui*w_1_nine_half - w_1_indep );
 614                                         oddPart = omegaOdd*(0.5*(pdf_E - pdf_W) - ui*w_1_x3 );
 615                                         dst[I(x, y, z, D3Q19_E)]  = pdf_E - evenPart - oddPart;
 616                                         dst[I(x, y, z, D3Q19_W)]  = pdf_W - evenPart + oddPart;
 617
 618                                         ui = uz;
 619                                         evenPart = omegaEven*( 0.5*(pdf_T + pdf_B) - ui*ui*w_1_nine_half - w_1_indep );
 620                                         oddPart = omegaOdd*(0.5*(pdf_T - pdf_B) - ui*w_1_x3 );
 621                                         dst[I(x, y, z, D3Q19_T)]  = pdf_T - evenPart - oddPart;
 622                                         dst[I(x, y, z, D3Q19_B)]  = pdf_B - evenPart + oddPart;
 623
 624                                         // direction: w_2
 625                                         w_2_indep = w_2*dir_indep_trm;
 626
 627                                         ui = -ux + uy;
 628                                         evenPart = omegaEven*( 0.5*(pdf_NW + pdf_SE) - ui*ui*w_2_nine_half - w_2_indep );
 629                                         oddPart = omegaOdd*(0.5*(pdf_NW - pdf_SE) - ui*w_2_x3 );
 630                                         dst[I(x, y, z, D3Q19_NW)] = pdf_NW - evenPart - oddPart;
 631                                         dst[I(x, y, z, D3Q19_SE)] = pdf_SE - evenPart + oddPart;
 632
 633                                         ui = ux + uy;
 634                                         evenPart = omegaEven*( 0.5*(pdf_NE + pdf_SW) - ui*ui*w_2_nine_half - w_2_indep );
 635                                         oddPart = omegaOdd*(0.5*(pdf_NE - pdf_SW) - ui*w_2_x3 );
 636                                         dst[I(x, y, z, D3Q19_NE)] = pdf_NE - evenPart - oddPart;
 637                                         dst[I(x, y, z, D3Q19_SW)] = pdf_SW - evenPart + oddPart;
 638
 639                                         ui = -ux + uz;
 640                                         evenPart = omegaEven*( 0.5*(pdf_TW + pdf_BE) - ui*ui*w_2_nine_half - w_2_indep );
 641                                         oddPart = omegaOdd*(0.5*(pdf_TW - pdf_BE) - ui*w_2_x3 );
 642                                         dst[I(x, y, z, D3Q19_TW)] = pdf_TW - evenPart - oddPart;
 643                                         dst[I(x, y, z, D3Q19_BE)] = pdf_BE - evenPart + oddPart;
 644
 645                                         ui = ux + uz;
 646                                         evenPart = omegaEven*( 0.5*(pdf_TE + pdf_BW) - ui*ui*w_2_nine_half - w_2_indep );
 647                                         oddPart = omegaOdd*(0.5*(pdf_TE - pdf_BW) - ui*w_2_x3 );
 648                                         dst[I(x, y, z, D3Q19_TE)] = pdf_TE - evenPart - oddPart;
 649                                         dst[I(x, y, z, D3Q19_BW)] = pdf_BW - evenPart + oddPart;
 650
 651                                         ui = -uy + uz;
 652                                         evenPart = omegaEven*( 0.5*(pdf_TS + pdf_BN) - ui*ui*w_2_nine_half - w_2_indep );
 653                                         oddPart = omegaOdd*(0.5*(pdf_TS - pdf_BN) - ui*w_2_x3 );
 654                                         dst[I(x, y, z, D3Q19_TS)] = pdf_TS - evenPart - oddPart;
 655                                         dst[I(x, y, z, D3Q19_BN)] = pdf_BN - evenPart + oddPart;
 656
 657                                         ui = uy + uz;
 658                                         evenPart = omegaEven*( 0.5*(pdf_TN + pdf_BS) - ui*ui*w_2_nine_half - w_2_indep );
 659                                         oddPart = omegaOdd*(0.5*(pdf_TN - pdf_BS) - ui*w_2_x3 );
 660                                         dst[I(x, y, z, D3Q19_TN)] = pdf_TN - evenPart - oddPart;
 661                                         dst[I(x, y, z, D3Q19_BS)] = pdf_BS - evenPart + oddPart;
 662
 663 #else
 664         #error No implementation for PROP_MODEL_NAME.
 665 #endif
 666
 667                                         #undef I
 668                                 }
 669                         }
 670                 } // z, y, x (from inner to outer)
 671                                 }
 672                         }
 673                 } // z, y, x (from inner to outer)
 674
 675                         X_LIKWID_STOP("blk-os");
 676                 } // parallel region
 677
 678                 // Stop counters before bounce back. Else computing loop balance will be incorrect.
 679
 680                 // Fixup bounce back PDFs.
 681                 #ifdef _OPENMP
 682                 #pragma omp parallel for default(none) \
 683                                 shared(kd, dst)
 684                 #endif
 685                 for (int i = 0; i < kd->nBounceBackPdfs; ++i) {
 686                         dst[kd->BounceBackPdfsDst[i]] = dst[kd->BounceBackPdfsSrc[i]];
 687                 }
 688
 689                 #ifdef VERIFICATION
 690                 kd->PdfsActive = dst;
 691                 KernelAddBodyForce(kd, ld, cd);
 692                 #endif
 693
 694                 #ifdef VTK_OUTPUT
 695
 696                 if (cd->VtkOutput && (iter % cd->VtkModulus) == 0) {
 697                         kd->PdfsActive = dst;
 698                         VtkWrite(ld, kd, cd, iter);
 699                 }
 700
 701                 #endif
 702
 703                 #ifdef STATISTICS
 704                 kd->PdfsActive = dst;
 705                 KernelStatistics(kd, ld, cd, iter);
 706                 #endif
 707
 708                 // swap grids
 709                 tmp = src;
 710                 src = dst;
 711                 dst = tmp;
 712
 713         } // for (int iter = 0; ...
 714
 715         #ifdef VTK_OUTPUT
 716
 717         if (cd->VtkOutput) {
 718                 kd->PdfsActive = src;
 719                 VtkWrite(ld, kd, cd, maxIterations);
 720         }
 721
 722         #endif
 723
 724         return;
 725 }
 726