#include "Memory.h"
#include "Vtk.h"
+#include "LikwidIf.h"
#include <inttypes.h>
#include <math.h>
Assert(kernelData != NULL);
Assert(cd != NULL);
- Assert(cd->Omega > 0.0);
- Assert(cd->Omega < 2.0);
+ Assert(cd->Omega > F(0.0));
+ Assert(cd->Omega < F(2.0));
KernelData * kd = (KernelData *)kernelData;
PdfT omega = cd->Omega;
PdfT omegaEven = omega;
-// PdfT omegaOdd = 8.0*((2.0-omegaEven)/(8.0-omegaEven)); //"standard" trt odd relaxation parameter
- PdfT magicParam = 1.0/12.0; // 1/4: best stability; 1/12: removes third-order advection error (best advection); 1/6: removes fourth-order diffusion error (best diffusion); 3/16: exact location of bounce back for poiseuille flow
- PdfT omegaOdd = 1.0/( 0.5 + magicParam/(1.0/omega - 0.5) );
+ // PdfT omegaOdd = 8.0*((F(2.0)-omegaEven)/(8.0-omegaEven)); //"standard" trt odd relaxation parameter
+ PdfT magicParam = F(1.0) / F(12.0);
+ // 1/ 4: best stability;
+ // 1/12: removes third-order advection error (best advection);
+ // 1/ 6: removes fourth-order diffusion error (best diffusion);
+ // 3/16: exact location of bounce back for poiseuille flow
+ PdfT omegaOdd = F(1.0) / (F(0.5) + magicParam / (F(1.0) / omega - F(0.5)));
- PdfT evenPart = 0.0;
- PdfT oddPart = 0.0;
- PdfT dir_indep_trm = 0.0;
+ PdfT evenPart = F(0.0);
+ PdfT oddPart = F(0.0);
+ PdfT dir_indep_trm = F(0.0);
- PdfT w_0 = 1.0 / 3.0;
- PdfT w_1 = 1.0 / 18.0;
- PdfT w_2 = 1.0 / 36.0;
+ PdfT w_0 = F(1.0) / F( 3.0);
+ PdfT w_1 = F(1.0) / F(18.0);
+ PdfT w_2 = F(1.0) / F(36.0);
- PdfT w_1_x3 = w_1 * 3.0; PdfT w_1_nine_half = w_1 * 9.0/2.0; PdfT w_1_indep = 0.0;
- PdfT w_2_x3 = w_2 * 3.0; PdfT w_2_nine_half = w_2 * 9.0/2.0; PdfT w_2_indep = 0.0;
+ PdfT w_1_x3 = w_1 * F(3.0); PdfT w_1_nine_half = w_1 * F(9.0)/F(2.0); PdfT w_1_indep = F(0.0);
+ PdfT w_2_x3 = w_2 * F(3.0); PdfT w_2_nine_half = w_2 * F(9.0)/F(2.0); PdfT w_2_indep = F(0.0);
PdfT ux, uy, uz, ui;
PdfT dens;
}
#endif
+ X_KERNEL_START(kernelData);
+
for (int iter = 0; iter < maxIterations; ++iter) {
+ X_LIKWID_START("os");
+
#ifdef _OPENMP
- #pragma omp parallel for collapse(3) default(none) \
+ #pragma omp parallel for collapse(2) default(none) \
shared(gDims,src, dst, w_0, w_1, w_2, omegaEven, omegaOdd, \
w_1_x3, w_2_x3, w_1_nine_half, w_2_nine_half, cd, \
oX, oY, oZ, nX, nY, nZ) \
pdf_B, pdf_BN, pdf_BE, pdf_BS, pdf_BW, \
evenPart, oddPart, w_1_indep, w_2_indep)
#endif
-
- for (int z = oZ; z < nZ + oZ; ++z) {
+ for (int x = oX; x < nX + oX; ++x) {
for (int y = oY; y < nY + oY; ++y) {
- for (int x = oX; x < nX + oX; ++x) {
+ #ifdef INTEL_OPT_DIRECTIVES
+ #pragma ivdep
+ #pragma vector always
+ #pragma simd
+ #endif
+ for (int z = oZ; z < nZ + oZ; ++z) { // LOOP os
#define I(x, y, z, dir) P_INDEX_5(gDims, (x), (y), (z), (dir))
#ifdef PROP_MODEL_PUSH
#ifdef LID_DRIVEN_CAVITY
if (z == nZ - 4 + oZ && x > 3 + oX && x < (nX - 4 + oX) && y > 3 + oY && y < (nY - 4 + oY)) {
- ux = 0.1 * 0.577;
- uy = 0.0;
- uz = 0.0;
+ ux = F(0.1 * 0.577);
+ uy = F(0.0);
+ uz = F(0.0);
} else {
#endif
pdf_T + pdf_TN + pdf_TE + pdf_TS + pdf_TW +
pdf_B + pdf_BN + pdf_BE + pdf_BS + pdf_BW;
- dir_indep_trm = dens - (ux * ux + uy * uy + uz * uz)*3.0/2.0;
+ dir_indep_trm = dens - (ux * ux + uy * uy + uz * uz) * F(3.0) / F(2.0);
#ifdef PROP_MODEL_PUSH
w_1_indep = w_1*dir_indep_trm;
ui = uy;
- evenPart = omegaEven*( 0.5*(pdf_N + pdf_S) - ui*ui*w_1_nine_half - w_1_indep );
- oddPart = omegaOdd*(0.5*(pdf_N - pdf_S) - ui*w_1_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_N + pdf_S) - ui*ui*w_1_nine_half - w_1_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_N - pdf_S) - ui*w_1_x3 );
dst[I(x, y + 1, z, D3Q19_N)] = pdf_N - evenPart - oddPart;
dst[I(x, y - 1, z, D3Q19_S)] = pdf_S - evenPart + oddPart;
ui = ux;
- evenPart = omegaEven*( 0.5*(pdf_E + pdf_W) - ui*ui*w_1_nine_half - w_1_indep );
- oddPart = omegaOdd*(0.5*(pdf_E - pdf_W) - ui*w_1_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_E + pdf_W) - ui*ui*w_1_nine_half - w_1_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_E - pdf_W) - ui*w_1_x3 );
dst[I(x + 1, y, z, D3Q19_E)] = pdf_E - evenPart - oddPart;
dst[I(x - 1, y, z, D3Q19_W)] = pdf_W - evenPart + oddPart;
ui = uz;
- evenPart = omegaEven*( 0.5*(pdf_T + pdf_B) - ui*ui*w_1_nine_half - w_1_indep );
- oddPart = omegaOdd*(0.5*(pdf_T - pdf_B) - ui*w_1_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_T + pdf_B) - ui*ui*w_1_nine_half - w_1_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_T - pdf_B) - ui*w_1_x3 );
dst[I(x, y, z + 1, D3Q19_T)] = pdf_T - evenPart - oddPart;
dst[I(x, y, z - 1, D3Q19_B)] = pdf_B - evenPart + oddPart;
w_2_indep = w_2*dir_indep_trm;
ui = -ux + uy;
- evenPart = omegaEven*( 0.5*(pdf_NW + pdf_SE) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_NW - pdf_SE) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_NW + pdf_SE) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_NW - pdf_SE) - ui*w_2_x3 );
dst[I(x - 1, y + 1, z, D3Q19_NW)] = pdf_NW - evenPart - oddPart;
dst[I(x + 1, y - 1, z, D3Q19_SE)] = pdf_SE - evenPart + oddPart;
ui = ux + uy;
- evenPart = omegaEven*( 0.5*(pdf_NE + pdf_SW) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_NE - pdf_SW) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_NE + pdf_SW) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_NE - pdf_SW) - ui*w_2_x3 );
dst[I(x + 1, y + 1, z, D3Q19_NE)] = pdf_NE - evenPart - oddPart;
dst[I(x - 1, y - 1, z, D3Q19_SW)] = pdf_SW - evenPart + oddPart;
ui = -ux + uz;
- evenPart = omegaEven*( 0.5*(pdf_TW + pdf_BE) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TW - pdf_BE) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TW + pdf_BE) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TW - pdf_BE) - ui*w_2_x3 );
dst[I(x - 1, y, z + 1, D3Q19_TW)] = pdf_TW - evenPart - oddPart;
dst[I(x + 1, y, z - 1, D3Q19_BE)] = pdf_BE - evenPart + oddPart;
ui = ux + uz;
- evenPart = omegaEven*( 0.5*(pdf_TE + pdf_BW) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TE - pdf_BW) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TE + pdf_BW) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TE - pdf_BW) - ui*w_2_x3 );
dst[I(x + 1, y, z + 1, D3Q19_TE)] = pdf_TE - evenPart - oddPart;
dst[I(x - 1, y, z - 1, D3Q19_BW)] = pdf_BW - evenPart + oddPart;
ui = -uy + uz;
- evenPart = omegaEven*( 0.5*(pdf_TS + pdf_BN) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TS - pdf_BN) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TS + pdf_BN) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TS - pdf_BN) - ui*w_2_x3 );
dst[I(x, y - 1, z + 1, D3Q19_TS)] = pdf_TS - evenPart - oddPart;
dst[I(x, y + 1, z - 1, D3Q19_BN)] = pdf_BN - evenPart + oddPart;
ui = uy + uz;
- evenPart = omegaEven*( 0.5*(pdf_TN + pdf_BS) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TN - pdf_BS) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TN + pdf_BS) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TN - pdf_BS) - ui*w_2_x3 );
dst[I(x, y + 1, z + 1, D3Q19_TN)] = pdf_TN - evenPart - oddPart;
dst[I(x, y - 1, z - 1, D3Q19_BS)] = pdf_BS - evenPart + oddPart;
w_1_indep = w_1*dir_indep_trm;
ui = uy;
- evenPart = omegaEven*( 0.5*(pdf_N + pdf_S) - ui*ui*w_1_nine_half - w_1_indep );
- oddPart = omegaOdd*(0.5*(pdf_N - pdf_S) - ui*w_1_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_N + pdf_S) - ui*ui*w_1_nine_half - w_1_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_N - pdf_S) - ui*w_1_x3 );
dst[I(x, y, z, D3Q19_N)] = pdf_N - evenPart - oddPart;
dst[I(x, y, z, D3Q19_S)] = pdf_S - evenPart + oddPart;
ui = ux;
- evenPart = omegaEven*( 0.5*(pdf_E + pdf_W) - ui*ui*w_1_nine_half - w_1_indep );
- oddPart = omegaOdd*(0.5*(pdf_E - pdf_W) - ui*w_1_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_E + pdf_W) - ui*ui*w_1_nine_half - w_1_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_E - pdf_W) - ui*w_1_x3 );
dst[I(x, y, z, D3Q19_E)] = pdf_E - evenPart - oddPart;
dst[I(x, y, z, D3Q19_W)] = pdf_W - evenPart + oddPart;
ui = uz;
- evenPart = omegaEven*( 0.5*(pdf_T + pdf_B) - ui*ui*w_1_nine_half - w_1_indep );
- oddPart = omegaOdd*(0.5*(pdf_T - pdf_B) - ui*w_1_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_T + pdf_B) - ui*ui*w_1_nine_half - w_1_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_T - pdf_B) - ui*w_1_x3 );
dst[I(x, y, z, D3Q19_T)] = pdf_T - evenPart - oddPart;
dst[I(x, y, z, D3Q19_B)] = pdf_B - evenPart + oddPart;
w_2_indep = w_2*dir_indep_trm;
ui = -ux + uy;
- evenPart = omegaEven*( 0.5*(pdf_NW + pdf_SE) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_NW - pdf_SE) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_NW + pdf_SE) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_NW - pdf_SE) - ui*w_2_x3 );
dst[I(x, y, z, D3Q19_NW)] = pdf_NW - evenPart - oddPart;
dst[I(x, y, z, D3Q19_SE)] = pdf_SE - evenPart + oddPart;
ui = ux + uy;
- evenPart = omegaEven*( 0.5*(pdf_NE + pdf_SW) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_NE - pdf_SW) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_NE + pdf_SW) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_NE - pdf_SW) - ui*w_2_x3 );
dst[I(x, y, z, D3Q19_NE)] = pdf_NE - evenPart - oddPart;
dst[I(x, y, z, D3Q19_SW)] = pdf_SW - evenPart + oddPart;
ui = -ux + uz;
- evenPart = omegaEven*( 0.5*(pdf_TW + pdf_BE) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TW - pdf_BE) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TW + pdf_BE) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TW - pdf_BE) - ui*w_2_x3 );
dst[I(x, y, z, D3Q19_TW)] = pdf_TW - evenPart - oddPart;
dst[I(x, y, z, D3Q19_BE)] = pdf_BE - evenPart + oddPart;
ui = ux + uz;
- evenPart = omegaEven*( 0.5*(pdf_TE + pdf_BW) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TE - pdf_BW) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TE + pdf_BW) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TE - pdf_BW) - ui*w_2_x3 );
dst[I(x, y, z, D3Q19_TE)] = pdf_TE - evenPart - oddPart;
dst[I(x, y, z, D3Q19_BW)] = pdf_BW - evenPart + oddPart;
ui = -uy + uz;
- evenPart = omegaEven*( 0.5*(pdf_TS + pdf_BN) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TS - pdf_BN) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TS + pdf_BN) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TS - pdf_BN) - ui*w_2_x3 );
dst[I(x, y, z, D3Q19_TS)] = pdf_TS - evenPart - oddPart;
dst[I(x, y, z, D3Q19_BN)] = pdf_BN - evenPart + oddPart;
ui = uy + uz;
- evenPart = omegaEven*( 0.5*(pdf_TN + pdf_BS) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TN - pdf_BS) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TN + pdf_BS) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TN - pdf_BS) - ui*w_2_x3 );
dst[I(x, y, z, D3Q19_TN)] = pdf_TN - evenPart - oddPart;
dst[I(x, y, z, D3Q19_BS)] = pdf_BS - evenPart + oddPart;
}
} // z, y, x (from inner to outer)
+ // Stop counters before bounce back. Else computing loop balance will be incorrect.
+ X_LIKWID_STOP("os");
+
+
// Fixup bounce back PDFs.
#ifdef _OPENMP
#pragma omp parallel for default(none) \
} // for (int iter = 0; ...
+ X_KERNEL_END(kernelData);
+
#ifdef VTK_OUTPUT
if (cd->VtkOutput) {
Assert(kernelData != NULL);
Assert(cd != NULL);
- Assert(cd->Omega > 0.0);
- Assert(cd->Omega < 2.0);
+ Assert(cd->Omega > F(0.0));
+ Assert(cd->Omega < F(2.0));
KernelData * kd = (KernelData *)kernelData;
PdfT omega = cd->Omega;
PdfT omegaEven = omega;
-// PdfT omegaOdd = 8.0*((2.0-omegaEven)/(8.0-omegaEven)); //"standard" trt odd relaxation parameter
- PdfT magicParam = 1.0/12.0; // 1/4: best stability; 1/12: removes third-order advection error (best advection); 1/6: removes fourth-order diffusion error (best diffusion); 3/16: exact location of bounce back for poiseuille flow
- PdfT omegaOdd = 1.0/( 0.5 + magicParam/(1.0/omega - 0.5) );
+ // PdfT omegaOdd = 8.0*((F(2.0)-omegaEven)/(8.0-omegaEven)); //"standard" trt odd relaxation parameter
+ PdfT magicParam = F(1.0)/F(12.0);
+ // 1/ 4: best stability;
+ // 1/12: removes third-order advection error (best advection);
+ // 1/ 6: removes fourth-order diffusion error (best diffusion);
+ // 3/16: exact location of bounce back for poiseuille flow
+ PdfT omegaOdd = F(1.0) / (F(0.5) + magicParam / (F(1.0) / omega - F(0.5)));
- PdfT evenPart = 0.0;
- PdfT oddPart = 0.0;
- PdfT dir_indep_trm = 0.0;
+ PdfT evenPart = F(0.0);
+ PdfT oddPart = F(0.0);
+ PdfT dir_indep_trm = F(0.0);
- PdfT w_0 = 1.0 / 3.0;
- PdfT w_1 = 1.0 / 18.0;
- PdfT w_2 = 1.0 / 36.0;
+ PdfT w_0 = F(1.0) / F( 3.0);
+ PdfT w_1 = F(1.0) / F(18.0);
+ PdfT w_2 = F(1.0) / F(36.0);
- PdfT w_1_x3 = w_1 * 3.0; PdfT w_1_nine_half = w_1 * 9.0/2.0; PdfT w_1_indep = 0.0;
- PdfT w_2_x3 = w_2 * 3.0; PdfT w_2_nine_half = w_2 * 9.0/2.0; PdfT w_2_indep = 0.0;
+ PdfT w_1_x3 = w_1 * F(3.0); PdfT w_1_nine_half = w_1 * F(9.0)/F(2.0); PdfT w_1_indep = F(0.0);
+ PdfT w_2_x3 = w_2 * F(3.0); PdfT w_2_nine_half = w_2 * F(9.0)/F(2.0); PdfT w_2_indep = F(0.0);
PdfT ux, uy, uz, ui;
PdfT dens;
nThreads = omp_get_max_threads();
#endif
+ X_KERNEL_START(kernelData);
+
for (int iter = 0; iter < maxIterations; ++iter) {
- // #ifdef _OPENMP --> add line continuation
- // #pragma omp parallel for collapse(3) default(none)
- // shared(gDims,src, dst, w_0, w_1, w_2, omegaEven, omegaOdd,
- // w_1_x3, w_2_x3, w_1_nine_half, w_2_nine_half, cd,
- // oX, oY, oZ, nX, nY, nZ, blk)
- // private(ux, uy, uz, ui, dens, dir_indep_trm,
- // pdf_C,
- // pdf_N, pdf_E, pdf_S, pdf_W,
- // pdf_NE, pdf_SE, pdf_SW, pdf_NW,
- // pdf_T, pdf_TN, pdf_TE, pdf_TS, pdf_TW,
- // pdf_B, pdf_BN, pdf_BE, pdf_BS, pdf_BW,
- // evenPart, oddPart, w_1_indep, w_2_indep)
- // #endif
#ifdef _OPENMP
- #pragma omp parallel for default(none) \
+ #pragma omp parallel default(none) \
shared(gDims,src, dst, w_0, w_1, w_2, omegaEven, omegaOdd, \
w_1_x3, w_2_x3, w_1_nine_half, w_2_nine_half, cd, \
oX, oY, oZ, nX, nY, nZ, blk, nThreads) \
pdf_B, pdf_BN, pdf_BE, pdf_BS, pdf_BW, \
evenPart, oddPart, w_1_indep, w_2_indep)
#endif
+ {
+ X_LIKWID_START("blk-os");
- for (int i = 0; i < nThreads; ++i) {
+ int threadId = omp_get_thread_num();
- int threadStartX = nX / nThreads * i;
- int threadEndX = nX / nThreads * (i + 1);
+ int threadStartX = nX / nThreads * threadId;
+ int threadEndX = nX / nThreads * (threadId + 1);
if (nX % nThreads > 0) {
- if (nX % nThreads > i) {
- threadStartX += i;
- threadEndX += i + 1;
+ if (nX % nThreads > threadId) {
+ threadStartX += threadId;
+ threadEndX += threadId + 1;
}
else {
threadStartX += nX % nThreads;
}
}
- // for (int z = oZ; z < nZ + oZ; ++z) {
- // for (int y = oY; y < nY + oY; ++y) {
- // for (int x = oX; x < nX + oX; ++x) {
- for (int bZ = oZ; bZ < nZ + oZ; bZ += blk[2]) {
+ for (int bX = oX + threadStartX; bX < threadEndX + oX; bX += blk[0]) {
for (int bY = oY; bY < nY + oY; bY += blk[1]) {
- for (int bX = oX + threadStartX; bX < threadEndX + oX; bX += blk[0]) {
- // for (int bX = oX; bX < nX + oX; bX += blk[0]) {
+ for (int bZ = oZ; bZ < nZ + oZ; bZ += blk[2]) {
// Must do everything here, else it would break collapse.
int eZ = MIN(bZ + blk[2], nZ + oZ);
int eY = MIN(bY + blk[1], nY + oY);
int eX = MIN(bX + blk[0], threadEndX + oX);
-// printf("%d: %d-%d %d-%d %d-%d %d - %d\n", omp_get_thread_num(), bZ, eZ, bY, eY, bX, eX, threadStartX, threadEndX);
-
- for (int z = bZ; z < eZ; ++z) {
+ for (int x = bX; x < eX; ++x) {
for (int y = bY; y < eY; ++y) {
- for (int x = bX; x < eX; ++x) {
+ #ifdef INTEL_OPT_DIRECTIVES
+ #pragma ivdep
+ #pragma vector always
+ #pragma simd
+ #endif
+ for (int z = bZ; z < eZ; ++z) {
#define I(x, y, z, dir) P_INDEX_5(gDims, (x), (y), (z), (dir))
// Load PDFs of local cell: pdf_N = src[I(x, y, z, D3Q19_N)]; ...
#define X(name, idx, idxinv, _x, _y, _z) JOIN(pdf_,name) = src[I(x, y, z, idx)];
- //if (isnan(JOIN(pdf_,name))) { printf("iter: %d %d %d %d %d %s nan\n", iter, x-oX, y-oY, z-oZ, idx, D3Q19_NAMES[idx]); exit(1);}
D3Q19_LIST
#undef X
// Load PDFs of local cell: pdf_N = src[I(x, y, z, D3Q19_N)]; ...
#define X(name, idx, idxinv, _x, _y, _z) JOIN(pdf_,name) = src[I(x - _x, y - _y, z - _z, idx)];
- //if (isnan(JOIN(pdf_,name))) { printf("iter: %d %d %d %d %d %s nan\n", iter, x-oX, y-oY, z-oZ, idx, D3Q19_NAMES[idx]); exit(1);}
D3Q19_LIST
#undef X
#ifdef LID_DRIVEN_CAVITY
if (z == nZ - 4 + oZ && x > 3 + oX && x < (nX - 4 + oX) && y > 3 + oY && y < (nY - 4 + oY)) {
- ux = 0.1 * 0.577;
+ ux = 0.1 * F(0.5)77;
uy = 0.0;
uz = 0.0;
} else {
#endif
ux = pdf_E + pdf_NE + pdf_SE + pdf_TE + pdf_BE -
- pdf_W - pdf_NW - pdf_SW - pdf_TW - pdf_BW;
+ pdf_W - pdf_NW - pdf_SW - pdf_TW - pdf_BW;
uy = pdf_N + pdf_NE + pdf_NW + pdf_TN + pdf_BN -
- pdf_S - pdf_SE - pdf_SW - pdf_TS - pdf_BS;
+ pdf_S - pdf_SE - pdf_SW - pdf_TS - pdf_BS;
uz = pdf_T + pdf_TE + pdf_TW + pdf_TN + pdf_TS -
- pdf_B - pdf_BE - pdf_BW - pdf_BN - pdf_BS;
+ pdf_B - pdf_BE - pdf_BW - pdf_BN - pdf_BS;
#ifdef LID_DRIVEN_CAVITY
}
pdf_T + pdf_TN + pdf_TE + pdf_TS + pdf_TW +
pdf_B + pdf_BN + pdf_BE + pdf_BS + pdf_BW;
- dir_indep_trm = dens - (ux * ux + uy * uy + uz * uz)*3.0/2.0;
+ dir_indep_trm = dens - (ux * ux + uy * uy + uz * uz) * F(3.0) / F(2.0);
#ifdef PROP_MODEL_PUSH
w_1_indep = w_1*dir_indep_trm;
ui = uy;
- evenPart = omegaEven*( 0.5*(pdf_N + pdf_S) - ui*ui*w_1_nine_half - w_1_indep );
- oddPart = omegaOdd*(0.5*(pdf_N - pdf_S) - ui*w_1_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_N + pdf_S) - ui*ui*w_1_nine_half - w_1_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_N - pdf_S) - ui*w_1_x3 );
dst[I(x, y + 1, z, D3Q19_N)] = pdf_N - evenPart - oddPart;
dst[I(x, y - 1, z, D3Q19_S)] = pdf_S - evenPart + oddPart;
ui = ux;
- evenPart = omegaEven*( 0.5*(pdf_E + pdf_W) - ui*ui*w_1_nine_half - w_1_indep );
- oddPart = omegaOdd*(0.5*(pdf_E - pdf_W) - ui*w_1_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_E + pdf_W) - ui*ui*w_1_nine_half - w_1_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_E - pdf_W) - ui*w_1_x3 );
dst[I(x + 1, y, z, D3Q19_E)] = pdf_E - evenPart - oddPart;
dst[I(x - 1, y, z, D3Q19_W)] = pdf_W - evenPart + oddPart;
ui = uz;
- evenPart = omegaEven*( 0.5*(pdf_T + pdf_B) - ui*ui*w_1_nine_half - w_1_indep );
- oddPart = omegaOdd*(0.5*(pdf_T - pdf_B) - ui*w_1_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_T + pdf_B) - ui*ui*w_1_nine_half - w_1_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_T - pdf_B) - ui*w_1_x3 );
dst[I(x, y, z + 1, D3Q19_T)] = pdf_T - evenPart - oddPart;
dst[I(x, y, z - 1, D3Q19_B)] = pdf_B - evenPart + oddPart;
w_2_indep = w_2*dir_indep_trm;
ui = -ux + uy;
- evenPart = omegaEven*( 0.5*(pdf_NW + pdf_SE) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_NW - pdf_SE) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_NW + pdf_SE) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_NW - pdf_SE) - ui*w_2_x3 );
dst[I(x - 1, y + 1, z, D3Q19_NW)] = pdf_NW - evenPart - oddPart;
dst[I(x + 1, y - 1, z, D3Q19_SE)] = pdf_SE - evenPart + oddPart;
ui = ux + uy;
- evenPart = omegaEven*( 0.5*(pdf_NE + pdf_SW) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_NE - pdf_SW) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_NE + pdf_SW) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_NE - pdf_SW) - ui*w_2_x3 );
dst[I(x + 1, y + 1, z, D3Q19_NE)] = pdf_NE - evenPart - oddPart;
dst[I(x - 1, y - 1, z, D3Q19_SW)] = pdf_SW - evenPart + oddPart;
ui = -ux + uz;
- evenPart = omegaEven*( 0.5*(pdf_TW + pdf_BE) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TW - pdf_BE) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TW + pdf_BE) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TW - pdf_BE) - ui*w_2_x3 );
dst[I(x - 1, y, z + 1, D3Q19_TW)] = pdf_TW - evenPart - oddPart;
dst[I(x + 1, y, z - 1, D3Q19_BE)] = pdf_BE - evenPart + oddPart;
ui = ux + uz;
- evenPart = omegaEven*( 0.5*(pdf_TE + pdf_BW) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TE - pdf_BW) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TE + pdf_BW) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TE - pdf_BW) - ui*w_2_x3 );
dst[I(x + 1, y, z + 1, D3Q19_TE)] = pdf_TE - evenPart - oddPart;
dst[I(x - 1, y, z - 1, D3Q19_BW)] = pdf_BW - evenPart + oddPart;
ui = -uy + uz;
- evenPart = omegaEven*( 0.5*(pdf_TS + pdf_BN) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TS - pdf_BN) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TS + pdf_BN) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TS - pdf_BN) - ui*w_2_x3 );
dst[I(x, y - 1, z + 1, D3Q19_TS)] = pdf_TS - evenPart - oddPart;
dst[I(x, y + 1, z - 1, D3Q19_BN)] = pdf_BN - evenPart + oddPart;
ui = uy + uz;
- evenPart = omegaEven*( 0.5*(pdf_TN + pdf_BS) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TN - pdf_BS) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TN + pdf_BS) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TN - pdf_BS) - ui*w_2_x3 );
dst[I(x, y + 1, z + 1, D3Q19_TN)] = pdf_TN - evenPart - oddPart;
dst[I(x, y - 1, z - 1, D3Q19_BS)] = pdf_BS - evenPart + oddPart;
w_1_indep = w_1*dir_indep_trm;
ui = uy;
- evenPart = omegaEven*( 0.5*(pdf_N + pdf_S) - ui*ui*w_1_nine_half - w_1_indep );
- oddPart = omegaOdd*(0.5*(pdf_N - pdf_S) - ui*w_1_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_N + pdf_S) - ui*ui*w_1_nine_half - w_1_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_N - pdf_S) - ui*w_1_x3 );
dst[I(x, y, z, D3Q19_N)] = pdf_N - evenPart - oddPart;
dst[I(x, y, z, D3Q19_S)] = pdf_S - evenPart + oddPart;
ui = ux;
- evenPart = omegaEven*( 0.5*(pdf_E + pdf_W) - ui*ui*w_1_nine_half - w_1_indep );
- oddPart = omegaOdd*(0.5*(pdf_E - pdf_W) - ui*w_1_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_E + pdf_W) - ui*ui*w_1_nine_half - w_1_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_E - pdf_W) - ui*w_1_x3 );
dst[I(x, y, z, D3Q19_E)] = pdf_E - evenPart - oddPart;
dst[I(x, y, z, D3Q19_W)] = pdf_W - evenPart + oddPart;
ui = uz;
- evenPart = omegaEven*( 0.5*(pdf_T + pdf_B) - ui*ui*w_1_nine_half - w_1_indep );
- oddPart = omegaOdd*(0.5*(pdf_T - pdf_B) - ui*w_1_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_T + pdf_B) - ui*ui*w_1_nine_half - w_1_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_T - pdf_B) - ui*w_1_x3 );
dst[I(x, y, z, D3Q19_T)] = pdf_T - evenPart - oddPart;
dst[I(x, y, z, D3Q19_B)] = pdf_B - evenPart + oddPart;
w_2_indep = w_2*dir_indep_trm;
ui = -ux + uy;
- evenPart = omegaEven*( 0.5*(pdf_NW + pdf_SE) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_NW - pdf_SE) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_NW + pdf_SE) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_NW - pdf_SE) - ui*w_2_x3 );
dst[I(x, y, z, D3Q19_NW)] = pdf_NW - evenPart - oddPart;
dst[I(x, y, z, D3Q19_SE)] = pdf_SE - evenPart + oddPart;
ui = ux + uy;
- evenPart = omegaEven*( 0.5*(pdf_NE + pdf_SW) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_NE - pdf_SW) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_NE + pdf_SW) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_NE - pdf_SW) - ui*w_2_x3 );
dst[I(x, y, z, D3Q19_NE)] = pdf_NE - evenPart - oddPart;
dst[I(x, y, z, D3Q19_SW)] = pdf_SW - evenPart + oddPart;
ui = -ux + uz;
- evenPart = omegaEven*( 0.5*(pdf_TW + pdf_BE) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TW - pdf_BE) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TW + pdf_BE) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TW - pdf_BE) - ui*w_2_x3 );
dst[I(x, y, z, D3Q19_TW)] = pdf_TW - evenPart - oddPart;
dst[I(x, y, z, D3Q19_BE)] = pdf_BE - evenPart + oddPart;
ui = ux + uz;
- evenPart = omegaEven*( 0.5*(pdf_TE + pdf_BW) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TE - pdf_BW) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TE + pdf_BW) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TE - pdf_BW) - ui*w_2_x3 );
dst[I(x, y, z, D3Q19_TE)] = pdf_TE - evenPart - oddPart;
dst[I(x, y, z, D3Q19_BW)] = pdf_BW - evenPart + oddPart;
ui = -uy + uz;
- evenPart = omegaEven*( 0.5*(pdf_TS + pdf_BN) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TS - pdf_BN) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TS + pdf_BN) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TS - pdf_BN) - ui*w_2_x3 );
dst[I(x, y, z, D3Q19_TS)] = pdf_TS - evenPart - oddPart;
dst[I(x, y, z, D3Q19_BN)] = pdf_BN - evenPart + oddPart;
ui = uy + uz;
- evenPart = omegaEven*( 0.5*(pdf_TN + pdf_BS) - ui*ui*w_2_nine_half - w_2_indep );
- oddPart = omegaOdd*(0.5*(pdf_TN - pdf_BS) - ui*w_2_x3 );
+ evenPart = omegaEven*( F(0.5)*(pdf_TN + pdf_BS) - ui*ui*w_2_nine_half - w_2_indep );
+ oddPart = omegaOdd*(F(0.5)*(pdf_TN - pdf_BS) - ui*w_2_x3 );
dst[I(x, y, z, D3Q19_TN)] = pdf_TN - evenPart - oddPart;
dst[I(x, y, z, D3Q19_BS)] = pdf_BS - evenPart + oddPart;
}
} // z, y, x (from inner to outer)
- } // loop over threads
+ X_LIKWID_STOP("blk-os");
+ } // parallel region
+
+ // Stop counters before bounce back. Else computing loop balance will be incorrect.
// Fixup bounce back PDFs.
#ifdef _OPENMP
} // for (int iter = 0; ...
+ X_KERNEL_END(kernelData);
+
#ifdef VTK_OUTPUT
if (cd->VtkOutput) {