[LbmBenchmarkKernelsPublic.git] / src / Makefile

# --------------------------------------------------------------------------
#
# Copyright
#   Markus Wittmann, 2016-2017
#   RRZE, University of Erlangen-Nuremberg, Germany
#   markus.wittmann -at- fau.de or hpc -at- rrze.fau.de
#
#   Viktor Haag, 2016
#   LSS, University of Erlangen-Nuremberg, Germany
#
#   Michael Hussnaetter, 2017-2018
#   University of Erlangen-Nuremberg, Germany
#   michael.hussnaetter -at- fau.de
#
#  This file is part of the Lattice Boltzmann Benchmark Kernels (LbmBenchKernels).
#
#  LbmBenchKernels is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  LbmBenchKernels is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with LbmBenchKernels.  If not, see <http://www.gnu.org/licenses/>.
#
# --------------------------------------------------------------------------

# Default configuration to chose.
# CONFIG ?= linux-gcc
CONFIG ?= linux-intel

BUILD ?= release

BENCHMARK ?= on

# If we compile for BENCHMARK all verifcation and statistics are disabled,
# if not specified otherwise.
ifeq (on,$(BENCHMARK))
  VERIFICATION ?= off
  VTK_OUTPUT   ?= off
  STATISTICS   ?= off
else
  VERIFICATION ?= on
  STATISTICS   ?= on
  VTK_OUTPUT   ?= on
endif


COLOR ?= on

OPENMP ?= on

VTK_OUTPUT_ASCII ?= off

LID_DRIVEN_CAVITY ?= off

ISA ?= avx

LIKWID ?= off

# Which floating point precision to use: dp (double precision) or sp (single preicision)
PRECISION ?= dp

# Global settings for the Makefile
SHELL      = sh

ifdef V
  Q        =
else
  Q        = @
endif

AR         = ar
CMP        = cmp
CHMOD      = chmod
CPP        = cpp
DIFF       = diff
ECHO       = echo
ECHO_E     = echo -e
ECHO_N     = echo -n
GIT        = git
GREP       = grep
MKDIR      = mkdir
MKDIR_P    = mkdir -p
NM         = nm
RM         = rm
RM_F       = rm -f
RM_RF      = rm -rf
TAR        = tar
TAR_CZF    = tar czf
TOUCH      = touch
SED        = sed


# Where to store objects and dependency files.
OBJECT_DIR      = obj/$(CONFIG)-$(BUILD)$(PREC)$(TAG)
DEP_DIR         = obj/$(CONFIG)-$(BUILD)$(PREC)$(TAG)-dep

# Sources to consider.
SOURCES_C = Main.c Memory.c Geometry.c Kernel.c \
            Vtk.c Pinning.c Padding.c Lattice.c

# ------------------------------------------------------------------------
# NO CHANGE BELOW SHOULD BE NEEDED
# ------------------------------------------------------------------------

OBJ_C = $(foreach SOURCE,$(SOURCES_C),$(OBJECT_DIR)/$(SOURCE:%.c=%.o)) \
		$(OBJECT_DIR)/BenchKernelD3Q19Common_PullSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19Common_PullAoS.o \
		$(OBJECT_DIR)/BenchKernelD3Q19Common_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19Common_PushAoS.o \
		$(OBJECT_DIR)/BenchKernelD3Q19_PullSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19_PullAoS.o \
		$(OBJECT_DIR)/BenchKernelD3Q19_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19_PushAoS.o \
		$(OBJECT_DIR)/BenchKernelD3Q19List_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListCommon_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19List_PushAoS.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListCommon_PushAoS.o \
		$(OBJECT_DIR)/BenchKernelD3Q19List_PullSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListCommon_PullSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19List_PullAoS.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListCommon_PullAoS.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAa_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAaCommon_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAa_PushAoS.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAaCommon_PushAoS.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAaRia_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAaRiaCommon_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAaPv_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvCommon_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListPullSplitNt_PullSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListPullSplitNtCommon_PullSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19Aa_AaAoS.o \
		$(OBJECT_DIR)/BenchKernelD3Q19AaCommon_AaAoS.o \
		$(OBJECT_DIR)/BenchKernelD3Q19Aa_AaSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19AaCommon_AaSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19AaVec_AaSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19AaVecCommon_AaSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19AaVecSl_AaSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19AaVecSlCommon_AaSoA.o

# Kernels which are using AVX-512 intrinsics.
OBJ_C_ISA_AVX_512 = \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvGatherAoSoA_PushAoSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvGatherAoSoACommon_PushAoSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvGather_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvGatherCommon_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvGatherHybrid_PushSoA.o \
		$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvGatherHybridCommon_PushSoA.o

OBJ = $(OBJ_C)

include ./config.$(CONFIG).mk


# Colorize command lines.
ifeq (on,$(COLOR))
  COLOR_GRAY        =\033[1;30m
  COLOR_LIGHT_GRAY  =\033[0;37m
  COLOR_CYAN        =\033[0;36m
  COLOR_LIGHT_CYAN  =\033[1;36m
  COLOR_MAGENTA     =\033[0;35m
  COLOR_DARK_MAGENTA=\033[1;35m
  COLOR_NO          =\033[0m
  COLOR_RED         =\033[0;31m
  COLOR_DARK_RED    =\033[1;31m
  COLOR_GREEN       =\033[0;32m
  COLOR_DARK_GREEEN =\033[1;32m
else
  COLOR_GRAY        =
  COLOR_LIGHT_GRAY  =
  COLOR_CYAN        =
  COLOR_LIGHT_CYAN  =
  COLOR_MAGENTA     =
  COLOR_DARK_MAGENTA=
  COLOR_NO          =
  COLOR_RED         =
  COLOR_DARK_RED    =
  COLOR_GREEN       =
  COLOR_DARK_GREEEN =
endif

# Dependencies which are not source files, that require
# rebuilding the application.
REBUILD_DEPS = $(filter-out %.d,$(MAKEFILE_LIST)) $(OBJECT_DIR)/.target $(DEP_DIR)/.target

BUILD_CONFIG=

# Verify computed flow field
ifeq (on,$(VERIFICATION))
  PP_FLAGS	+= $(D)VERIFICATION
endif

ifeq (on,$(VTK_OUTPUT))
  PP_FLAGS += $(D)VTK_OUTPUT
endif

ifeq (on,$(STATISTICS))
  PP_FLAGS += $(D)STATISTICS
endif

ifeq (on,$(VTK_OUTPUT_ASCII))
  PP_FLAGS += $(D)VTK_OUTPUT_ASCII
endif

ifeq (on,$(LID_DRIVEN_CAVITY))
  PP_FLAGS += $(D)LID_DRIVEN_CAVITY
endif

ifeq (avx512,$(ISA))
  PP_FLAGS += $(D)VECTOR_AVX512
endif

ifeq (avx,$(ISA))
  PP_FLAGS += $(D)VECTOR_AVX
endif

ifeq (sse,$(ISA))
  PP_FLAGS += $(D)VECTOR_SSE
endif

# Required for posix_memalign
PP_FLAGS += $(D)_POSIX_C_SOURCE=200112L

ifeq (on,$(LIKWID))
  PP_FLAGS += $(LIKWID_INC) $(D)HAVE_LIKWID
  LD_LIBS  += $(LIKWID_LIB) -llikwid
endif

ifeq (HBM,$(ADJ_LIST_MEM_TYPE))
  #MEMKIND is required for this!
  MEMKIND      = on
  C_FLAGS     += -DALLOC_ADJ_LIST_IN_HBM=1
else
  C_FLAGS     += -DALLOC_ADJ_LIST_IN_HBM=0
endif

ifeq (HBM,$(PDF_MEM_TYPE))
  #MEMKIND is required for this!
  MEMKIND      = on
  C_FLAGS     += -DALLOC_PDF_IN_HBM=1
else
  C_FLAGS     += -DALLOC_PDF_IN_HBM=0
endif

ifeq (on, $(MEMKIND))
  C_FLAGS     += -DHAVE_MEMKIND
  LD_FLAGS    += -lmemkind
endif

# Prefetch settings
SOFTWARE_PREFETCH_LOOKAHEAD_L2 ?= 0
SOFTWARE_PREFETCH_LOOKAHEAD_L1 ?= 0
C_FLAGS                        += -DSOFTWARE_PREFETCH_LOOKAHEAD_L2=$(SOFTWARE_PREFETCH_LOOKAHEAD_L2) -DSOFTWARE_PREFETCH_LOOKAHEAD_L1=$(SOFTWARE_PREFETCH_LOOKAHEAD_L1)


ifeq (dp,$(PRECISION))
  PP_FLAGS += $(D)PRECISION_DP
  PREC=-dp
else
ifeq (sp,$(PRECISION))
  PP_FLAGS += $(D)PRECISION_SP
  PREC=-sp
else
  $(error PRECISION is only be allowed to be sp (single precision) or dp (doble precision))
endif
endif

# Use AVX512 intrinsic kernels only when ISA=avx512 and we have double
# precision support.
ifeq (avx512,$(ISA))
  ifeq (dp,$(PRECISION))
    OBJ_C += $(OBJ_C_ISA_AVX_512)
    PP_FLAGS += $(D)VECTOR_AVX512_GATHER
  endif
endif

# ARCH can only be assigned a string without a space. The space is escaped as
# a comma which we have to replace here.

ifdef TARCH
  ARCH_W_COMMA := $(TARCH)

  COMMA_ := ,
  override TARCH := $(subst $(COMMA_), ,$(TARCH))

  $(info $(shell echo -e "\n$(COLOR_GREEN)INFO: Automatically expanding comma in TARCH variable:  TARCH=$(TARCH).$(COLOR_NO)\n"))

endif

.phony: all clean clean-all

#$(info $(shell $(ECHO_E) "# Configuration:   CONFIG=$(COLOR_CYAN)$(CONFIG)$(COLOR_NO)   BUILD=$(COLOR_CYAN)$(BUILD)$(COLOR_NO)   VERIFICATION=$(COLOR_CYAN)$(VERIFICATION)$(COLOR_NO)   STATISTICS=$(COLOR_CYAN)$(STATISTICS)$(COLOR_NO)   VTK_OUTPUT=$(COLOR_CYAN)$(VTK_OUTPUT)$(COLOR_NO)  OPENMP=$(COLOR_CYAN)$(OPENMP)$(COLOR_NO)   ISA=$(COLOR_CYAN)$(ISA)$(COLOR_NO)   LIKWID=$(COLOR_CYAN)$(LIKWID)$(COLOR_NO)   TARCH=$(COLOR_CYAN)$(TARCH)$(COLOR_NO)   building $(.DEFAULT_GOAL)..."))

$(info $(shell $(ECHO_E) "# Configuration: CONFIG=$(COLOR_CYAN)$(CONFIG)$(COLOR_NO)   BUILD=$(COLOR_CYAN)$(BUILD)$(COLOR_NO)  PRECISION=$(COLOR_CYAN)$(PRECISION)$(COLOR_NO)"))
$(info $(shell $(ECHO_E) "#                OPENMP=$(COLOR_CYAN)$(OPENMP)$(COLOR_NO)   ISA=$(COLOR_CYAN)$(ISA)$(COLOR_NO)   LIKWID=$(COLOR_CYAN)$(LIKWID)$(COLOR_NO)"))
$(info $(shell $(ECHO_E) "#                VERIFICATION=$(COLOR_CYAN)$(VERIFICATION)$(COLOR_NO)   STATISTICS=$(COLOR_CYAN)$(STATISTICS)$(COLOR_NO)   VTK_OUTPUT=$(COLOR_CYAN)$(VTK_OUTPUT)$(COLOR_NO)"))
$(info $(shell $(ECHO_E) "#                      target=$(.DEFAULT_GOAL)"))

$(info # Object dir:      $(OBJECT_DIR))
$(info # Dependency dir:  $(DEP_DIR))
# $(info # objects:        $(OBJ))

BIN_DIR=../bin

all: $(BIN_DIR)/lbmbenchk-$(CONFIG)-$(BUILD)$(BUILD_CONFIG)$(PREC)$(TAG)


# ------------------------------------------------------------------------
# TODO: Include dependencies, if they exist and no clean target is specified.

#ifeq (,$(SKIP_DEP_INCLUDE))
#  ifneq (,$(OBJ))
#    TMP_OBJ__ = $(patsubst $(OBJECT_DIR)/%,$(DEP_DIR)/%,$(OBJ))
#    $(warning TMP_OBJ__: $(TMP_OBJ__))
#    $(warning OBJ: $(OBJ))
#    $(warning DEPS:      $(patsubst %.o,%.d,$(TMP_OBJ__)))
#    $(warning $(patsubst $(OBJECT_DIR)/%.o,$(DEP_DIR)/%.d,$(OBJ)))
    -include $(patsubst $(OBJECT_DIR)/%.o,$(DEP_DIR)/%.d,$(OBJ))
#  endif
#endif


$(BIN_DIR):
	[ -d "$@" ] || mkdir -p "$@"

$(BIN_DIR)/lbmbenchk-$(CONFIG)-$(BUILD)$(BUILD_CONFIG)$(PREC)$(TAG): $(OBJ) $(REBUILD_DEPS) $(DEP_DIR)/.target | $(BIN_DIR)
	@$(ECHO_E) "linking: $(COLOR_CYAN)$@$(COLOR_NO)"
	$(LD) $(LD_FLAGS) -o $@ $(filter-out $(REBUILD_DEPS),$^) $(LD_LIBS)
	@$(ECHO_E) "# Builded binary:      $(COLOR_CYAN)$@$(COLOR_NO)"
	@$(ECHO_E) "# Configuration was:   CONFIG=$(COLOR_CYAN)$(CONFIG)$(COLOR_NO)   BUILD=$(COLOR_CYAN)$(BUILD)$(COLOR_NO)  PRECISION=$(COLOR_CYAN)$(PRECISION)$(COLOR_NO)"
	@$(ECHO_E) "#                      OPENMP=$(COLOR_CYAN)$(OPENMP)$(COLOR_NO)   ISA=$(COLOR_CYAN)$(ISA)$(COLOR_NO)   LIKWID=$(COLOR_CYAN)$(LIKWID)$(COLOR_NO)"
	@$(ECHO_E) "#                      VERIFICATION=$(COLOR_CYAN)$(VERIFICATION)$(COLOR_NO)   STATISTICS=$(COLOR_CYAN)$(STATISTICS)$(COLOR_NO)   VTK_OUTPUT=$(COLOR_CYAN)$(VTK_OUTPUT)$(COLOR_NO)"
	@$(ECHO_E) "#                      target=$(.DEFAULT_GOAL)"

$(OBJECT_DIR)/%_SoA.o: %.c $(REBUILD_DEPS)
	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO)    $(COLOR_MAGENTA)DATA_LAYOUT_SOA$(COLOR_NO)"
	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_SOA -c $< -o $@

$(OBJECT_DIR)/%_PushSoA.o: %.c $(REBUILD_DEPS)
	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO)    $(COLOR_MAGENTA)DATA_LAYOUT_SOA$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_PUSH$(COLOR_NO)"
	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_SOA $(D)PROP_MODEL_PUSH -c $< -o $@

$(OBJECT_DIR)/%_PushAoS.o: %.c $(REBUILD_DEPS)
	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO)    $(COLOR_MAGENTA)DATA_LAYOUT_AOS$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_PUSH$(COLOR_NO)"
	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_AOS $(D)PROP_MODEL_PUSH -c $< -o $@

$(OBJECT_DIR)/%_PushAoSoA.o: %.c $(REBUILD_DEPS)
	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO)    $(COLOR_MAGENTA)DATA_LAYOUT_AOSOA$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_PUSH$(COLOR_NO)"
	$(Q)$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_AOSOA $(D)PROP_MODEL_PUSH -c $< -o $@

$(OBJECT_DIR)/%_PullSoA.o: %.c $(REBUILD_DEPS)
	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO)    $(COLOR_MAGENTA)DATA_LAYOUT_SOA$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_PULL$(COLOR_NO)"
	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_SOA $(D)PROP_MODEL_PULL -c $< -o $@

$(OBJECT_DIR)/%_PullAoS.o: %.c $(REBUILD_DEPS)
	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO)    $(COLOR_MAGENTA)DATA_LAYOUT_AOS$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_PULL$(COLOR_NO)"
	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_AOS $(D)PROP_MODEL_PULL -c $< -o $@

$(OBJECT_DIR)/%_AoS.o: %.c $(REBUILD_DEPS)
	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO)    $(COLOR_MAGENTA)DATA_LAYOUT_AOS$(COLOR_NO)"
	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_AOS -c $< -o $@

$(OBJECT_DIR)/%_AaAoS.o: %.c $(REBUILD_DEPS)
	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO)    $(COLOR_MAGENTA)DATA_LAYOUT_AOS$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_AA$(COLOR_NO)"
	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_AOS $(D)PROP_MODEL_AA -c $< -o $@

$(OBJECT_DIR)/%_AaSoA.o: %.c $(REBUILD_DEPS)
	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO)    $(COLOR_MAGENTA)DATA_LAYOUT_SOA$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_AA$(COLOR_NO)"
	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_SOA $(D)PROP_MODEL_AA -c $< -o $@

$(OBJECT_DIR)/%.o: %.c $(REBUILD_DEPS)
	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO)"
	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) -c $< -o $@


# ------------------------------------------------------------------------
# Following targets in principal make only sure, that the directories
# are created if the do not exist already.

$(DEP_DIR)/.target:
	$(Q)$(MKDIR_P) $(DEP_DIR)
	$(Q)$(TOUCH) $@

# ------------------------------------------------------------------------
# Current configuration.
MAKE_CFG = SYSTEM=$(SYSTEM) // BUILD=$(BUILD) // MAKEOVERRIDES=\"$(strip $(MAKEOVERRIDES))\" // VERIFICATION=$(VERIFICATION) // STATISTICS=$(STATISTICS) // VTK_OUTPUT=$(VTK_OUTPUT) // VTK_OUTPUT_ASCII=$(VTK_OUTPUT_ASCII) // LID_DRIVEN_CAVITY=$(LID_DRIVEN_CAVITY) // ISA=$(ISA) // LIKWID=$(LIKWID) // PRECISION=$(PRECISION)

# Compare current configuration to the last one so we know when to
# rebuild this system/target despite when sources have not changed.
$(OBJECT_DIR)/.target::
	$(Q)$(MKDIR_P) $(OBJECT_DIR)
	@ ( [ -f $(OBJECT_DIR)/.target ] && $(ECHO) $(MAKE_CFG) | $(CMP) -s $(OBJECT_DIR)/.target - )           \
       || ( $(ECHO) $(MAKE_CFG) > $(OBJECT_DIR)/.target; $(ECHO) "# configuration variables changed, rebuilding ..." )


clean:
	-rm $(OBJECT_DIR)/*.o $(DEP_DIR)/*.d

clean-all:
	-[ -d obj ] && rm -r obj
Commit	Line	Data
10988083 MW	1	# --------------------------------------------------------------------------
	2	#
	3	# Copyright
	4	# Markus Wittmann, 2016-2017
	5	# RRZE, University of Erlangen-Nuremberg, Germany
	6	# markus.wittmann -at- fau.de or hpc -at- rrze.fau.de
	7	#
	8	# Viktor Haag, 2016
	9	# LSS, University of Erlangen-Nuremberg, Germany
	10	#
8cafd9ea MW	11	# Michael Hussnaetter, 2017-2018
	12	# University of Erlangen-Nuremberg, Germany
	13	# michael.hussnaetter -at- fau.de
	14	#
10988083 MW	15	# This file is part of the Lattice Boltzmann Benchmark Kernels (LbmBenchKernels).
	16	#
	17	# LbmBenchKernels is free software: you can redistribute it and/or modify
	18	# it under the terms of the GNU General Public License as published by
	19	# the Free Software Foundation, either version 3 of the License, or
	20	# (at your option) any later version.
	21	#
	22	# LbmBenchKernels is distributed in the hope that it will be useful,
	23	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	24	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	25	# GNU General Public License for more details.
	26	#
	27	# You should have received a copy of the GNU General Public License
	28	# along with LbmBenchKernels. If not, see <http://www.gnu.org/licenses/>.
	29	#
	30	# --------------------------------------------------------------------------
	31
	32	# Default configuration to chose.
	33	# CONFIG ?= linux-gcc
	34	CONFIG ?= linux-intel
	35
e3f82424 MW	36	BUILD ?= release
	37
	38	BENCHMARK ?= on
10988083 MW	39
	40	# If we compile for BENCHMARK all verifcation and statistics are disabled,
	41	# if not specified otherwise.
	42	ifeq (on,$(BENCHMARK))
	43	VERIFICATION ?= off
	44	VTK_OUTPUT ?= off
	45	STATISTICS ?= off
e3f82424 MW	46	else
	47	VERIFICATION ?= on
	48	STATISTICS ?= on
	49	VTK_OUTPUT ?= on
10988083 MW	50	endif
10988083 MW	51
10988083 MW	52
	53	COLOR ?= on
	54
	55	OPENMP ?= on
	56
	57	VTK_OUTPUT_ASCII ?= off
	58
	59	LID_DRIVEN_CAVITY ?= off
	60
	61	ISA ?= avx
	62
	63	LIKWID ?= off
	64
0fde6e45 MW	65	# Which floating point precision to use: dp (double precision) or sp (single preicision)
	66	PRECISION ?= dp
	67
10988083 MW	68	# Global settings for the Makefile
	69	SHELL = sh
	70
	71	ifdef V
	72	Q =
	73	else
	74	Q = @
	75	endif
	76
	77	AR = ar
	78	CMP = cmp
	79	CHMOD = chmod
	80	CPP = cpp
	81	DIFF = diff
	82	ECHO = echo
	83	ECHO_E = echo -e
	84	ECHO_N = echo -n
	85	GIT = git
	86	GREP = grep
	87	MKDIR = mkdir
	88	MKDIR_P = mkdir -p
	89	NM = nm
	90	RM = rm
	91	RM_F = rm -f
	92	RM_RF = rm -rf
	93	TAR = tar
	94	TAR_CZF = tar czf
	95	TOUCH = touch
	96	SED = sed
	97
	98
	99
	100	# Where to store objects and dependency files.
0fde6e45 MW	101	OBJECT_DIR = obj/$(CONFIG)-$(BUILD)$(PREC)$(TAG)
0fde6e45 MW	102	DEP_DIR = obj/$(CONFIG)-$(BUILD)$(PREC)$(TAG)-dep
10988083 MW	103
	104	# Sources to consider.
	105	SOURCES_C = Main.c Memory.c Geometry.c Kernel.c \
e3f82424	106	Vtk.c Pinning.c Padding.c Lattice.c
10988083 MW	107
	108	# ------------------------------------------------------------------------
	109	# NO CHANGE BELOW SHOULD BE NEEDED
	110	# ------------------------------------------------------------------------
	111
	112	OBJ_C = $(foreach SOURCE,$(SOURCES_C),$(OBJECT_DIR)/$(SOURCE:%.c=%.o)) \
	113	$(OBJECT_DIR)/BenchKernelD3Q19Common_PullSoA.o \
	114	$(OBJECT_DIR)/BenchKernelD3Q19Common_PullAoS.o \
	115	$(OBJECT_DIR)/BenchKernelD3Q19Common_PushSoA.o \
	116	$(OBJECT_DIR)/BenchKernelD3Q19Common_PushAoS.o \
	117	$(OBJECT_DIR)/BenchKernelD3Q19_PullSoA.o \
	118	$(OBJECT_DIR)/BenchKernelD3Q19_PullAoS.o \
	119	$(OBJECT_DIR)/BenchKernelD3Q19_PushSoA.o \
	120	$(OBJECT_DIR)/BenchKernelD3Q19_PushAoS.o \
	121	$(OBJECT_DIR)/BenchKernelD3Q19List_PushSoA.o \
	122	$(OBJECT_DIR)/BenchKernelD3Q19ListCommon_PushSoA.o \
	123	$(OBJECT_DIR)/BenchKernelD3Q19List_PushAoS.o \
	124	$(OBJECT_DIR)/BenchKernelD3Q19ListCommon_PushAoS.o \
	125	$(OBJECT_DIR)/BenchKernelD3Q19List_PullSoA.o \
	126	$(OBJECT_DIR)/BenchKernelD3Q19ListCommon_PullSoA.o \
	127	$(OBJECT_DIR)/BenchKernelD3Q19List_PullAoS.o \
	128	$(OBJECT_DIR)/BenchKernelD3Q19ListCommon_PullAoS.o \
	129	$(OBJECT_DIR)/BenchKernelD3Q19ListAa_PushSoA.o \
	130	$(OBJECT_DIR)/BenchKernelD3Q19ListAaCommon_PushSoA.o \
	131	$(OBJECT_DIR)/BenchKernelD3Q19ListAa_PushAoS.o \
	132	$(OBJECT_DIR)/BenchKernelD3Q19ListAaCommon_PushAoS.o \
	133	$(OBJECT_DIR)/BenchKernelD3Q19ListAaRia_PushSoA.o \
	134	$(OBJECT_DIR)/BenchKernelD3Q19ListAaRiaCommon_PushSoA.o \
	135	$(OBJECT_DIR)/BenchKernelD3Q19ListAaPv_PushSoA.o \
	136	$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvCommon_PushSoA.o \
	137	$(OBJECT_DIR)/BenchKernelD3Q19ListPullSplitNt_PullSoA.o \
e3f82424 MW	138	$(OBJECT_DIR)/BenchKernelD3Q19ListPullSplitNtCommon_PullSoA.o \
	139	$(OBJECT_DIR)/BenchKernelD3Q19Aa_AaAoS.o \
	140	$(OBJECT_DIR)/BenchKernelD3Q19AaCommon_AaAoS.o \
	141	$(OBJECT_DIR)/BenchKernelD3Q19Aa_AaSoA.o \
	142	$(OBJECT_DIR)/BenchKernelD3Q19AaCommon_AaSoA.o \
	143	$(OBJECT_DIR)/BenchKernelD3Q19AaVec_AaSoA.o \
0fde6e45 MW	144	$(OBJECT_DIR)/BenchKernelD3Q19AaVecCommon_AaSoA.o \
	145	$(OBJECT_DIR)/BenchKernelD3Q19AaVecSl_AaSoA.o \
	146	$(OBJECT_DIR)/BenchKernelD3Q19AaVecSlCommon_AaSoA.o
10988083	147
8cafd9ea MW	148	# Kernels which are using AVX-512 intrinsics.
	149	OBJ_C_ISA_AVX_512 = \
	150	$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvGatherAoSoA_PushAoSoA.o \
	151	$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvGatherAoSoACommon_PushAoSoA.o \
	152	$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvGather_PushSoA.o \
	153	$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvGatherCommon_PushSoA.o \
	154	$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvGatherHybrid_PushSoA.o \
	155	$(OBJECT_DIR)/BenchKernelD3Q19ListAaPvGatherHybridCommon_PushSoA.o
	156
10988083 MW	157	OBJ = $(OBJ_C)
	158
	159	include ./config.$(CONFIG).mk
	160
	161
	162	# Colorize command lines.
	163	ifeq (on,$(COLOR))
	164	COLOR_GRAY =\033[1;30m
	165	COLOR_LIGHT_GRAY =\033[0;37m
	166	COLOR_CYAN =\033[0;36m
	167	COLOR_LIGHT_CYAN =\033[1;36m
	168	COLOR_MAGENTA =\033[0;35m
	169	COLOR_DARK_MAGENTA=\033[1;35m
	170	COLOR_NO =\033[0m
	171	COLOR_RED =\033[0;31m
	172	COLOR_DARK_RED =\033[1;31m
	173	COLOR_GREEN =\033[0;32m
	174	COLOR_DARK_GREEEN =\033[1;32m
	175	else
	176	COLOR_GRAY =
	177	COLOR_LIGHT_GRAY =
	178	COLOR_CYAN =
	179	COLOR_LIGHT_CYAN =
	180	COLOR_MAGENTA =
	181	COLOR_DARK_MAGENTA=
	182	COLOR_NO =
	183	COLOR_RED =
	184	COLOR_DARK_RED =
	185	COLOR_GREEN =
	186	COLOR_DARK_GREEEN =
	187	endif
	188
	189	# Dependencies which are not source files, that require
	190	# rebuilding the application.
	191	REBUILD_DEPS = $(filter-out %.d,$(MAKEFILE_LIST)) $(OBJECT_DIR)/.target $(DEP_DIR)/.target
	192
	193	BUILD_CONFIG=
	194
	195	# Verify computed flow field
	196	ifeq (on,$(VERIFICATION))
	197	PP_FLAGS += $(D)VERIFICATION
	198	endif
	199
	200	ifeq (on,$(VTK_OUTPUT))
	201	PP_FLAGS += $(D)VTK_OUTPUT
	202	endif
	203
	204	ifeq (on,$(STATISTICS))
	205	PP_FLAGS += $(D)STATISTICS
	206	endif
	207
	208	ifeq (on,$(VTK_OUTPUT_ASCII))
	209	PP_FLAGS += $(D)VTK_OUTPUT_ASCII
	210	endif
	211
	212	ifeq (on,$(LID_DRIVEN_CAVITY))
	213	PP_FLAGS += $(D)LID_DRIVEN_CAVITY
	214	endif
	215
8cafd9ea MW	216	ifeq (avx512,$(ISA))
	217	PP_FLAGS += $(D)VECTOR_AVX512
	218	endif
	219
10988083 MW	220	ifeq (avx,$(ISA))
	221	PP_FLAGS += $(D)VECTOR_AVX
	222	endif
8cafd9ea	223
10988083 MW	224	ifeq (sse,$(ISA))
	225	PP_FLAGS += $(D)VECTOR_SSE
	226	endif
	227
	228	# Required for posix_memalign
	229	PP_FLAGS += $(D)_POSIX_C_SOURCE=200112L
	230
	231	ifeq (on,$(LIKWID))
	232	PP_FLAGS += $(LIKWID_INC) $(D)HAVE_LIKWID
	233	LD_LIBS += $(LIKWID_LIB) -llikwid
	234	endif
	235
8cafd9ea MW	236	ifeq (HBM,$(ADJ_LIST_MEM_TYPE))
	237	#MEMKIND is required for this!
	238	MEMKIND = on
	239	C_FLAGS += -DALLOC_ADJ_LIST_IN_HBM=1
	240	else
	241	C_FLAGS += -DALLOC_ADJ_LIST_IN_HBM=0
	242	endif
	243
	244	ifeq (HBM,$(PDF_MEM_TYPE))
	245	#MEMKIND is required for this!
	246	MEMKIND = on
	247	C_FLAGS += -DALLOC_PDF_IN_HBM=1
	248	else
	249	C_FLAGS += -DALLOC_PDF_IN_HBM=0
	250	endif
	251
	252	ifeq (on, $(MEMKIND))
	253	C_FLAGS += -DHAVE_MEMKIND
	254	LD_FLAGS += -lmemkind
	255	endif
	256
	257	# Prefetch settings
	258	SOFTWARE_PREFETCH_LOOKAHEAD_L2 ?= 0
	259	SOFTWARE_PREFETCH_LOOKAHEAD_L1 ?= 0
	260	C_FLAGS += -DSOFTWARE_PREFETCH_LOOKAHEAD_L2=$(SOFTWARE_PREFETCH_LOOKAHEAD_L2) -DSOFTWARE_PREFETCH_LOOKAHEAD_L1=$(SOFTWARE_PREFETCH_LOOKAHEAD_L1)
	261
0fde6e45 MW	262
	263	ifeq (dp,$(PRECISION))
	264	PP_FLAGS += $(D)PRECISION_DP
	265	PREC=-dp
	266	else
	267	ifeq (sp,$(PRECISION))
	268	PP_FLAGS += $(D)PRECISION_SP
	269	PREC=-sp
	270	else
	271	$(error PRECISION is only be allowed to be sp (single precision) or dp (doble precision))
	272	endif
	273	endif
	274
8cafd9ea MW	275	# Use AVX512 intrinsic kernels only when ISA=avx512 and we have double
	276	# precision support.
	277	ifeq (avx512,$(ISA))
	278	ifeq (dp,$(PRECISION))
	279	OBJ_C += $(OBJ_C_ISA_AVX_512)
9e0051cb	280	PP_FLAGS += $(D)VECTOR_AVX512_GATHER
8cafd9ea MW	281	endif
	282	endif
	283
e3f82424 MW	284	# ARCH can only be assigned a string without a space. The space is escaped as
	285	# a comma which we have to replace here.
	286
	287	ifdef TARCH
	288	ARCH_W_COMMA := $(TARCH)
	289
	290	COMMA_ := ,
	291	override TARCH := $(subst $(COMMA_), ,$(TARCH))
	292
	293	$(info $(shell echo -e "\n$(COLOR_GREEN)INFO: Automatically expanding comma in TARCH variable: TARCH=$(TARCH).$(COLOR_NO)\n"))
	294
	295	endif
	296
10988083 MW	297	.phony: all clean clean-all
10988083 MW	298
0fde6e45	299	#$(info $(shell $(ECHO_E) "# Configuration: CONFIG=$(COLOR_CYAN)$(CONFIG)$(COLOR_NO) BUILD=$(COLOR_CYAN)$(BUILD)$(COLOR_NO) VERIFICATION=$(COLOR_CYAN)$(VERIFICATION)$(COLOR_NO) STATISTICS=$(COLOR_CYAN)$(STATISTICS)$(COLOR_NO) VTK_OUTPUT=$(COLOR_CYAN)$(VTK_OUTPUT)$(COLOR_NO) OPENMP=$(COLOR_CYAN)$(OPENMP)$(COLOR_NO) ISA=$(COLOR_CYAN)$(ISA)$(COLOR_NO) LIKWID=$(COLOR_CYAN)$(LIKWID)$(COLOR_NO) TARCH=$(COLOR_CYAN)$(TARCH)$(COLOR_NO) building $(.DEFAULT_GOAL)..."))
10988083	300
0fde6e45 MW	301	$(info $(shell $(ECHO_E) "# Configuration: CONFIG=$(COLOR_CYAN)$(CONFIG)$(COLOR_NO) BUILD=$(COLOR_CYAN)$(BUILD)$(COLOR_NO) PRECISION=$(COLOR_CYAN)$(PRECISION)$(COLOR_NO)"))
	302	$(info $(shell $(ECHO_E) "# OPENMP=$(COLOR_CYAN)$(OPENMP)$(COLOR_NO) ISA=$(COLOR_CYAN)$(ISA)$(COLOR_NO) LIKWID=$(COLOR_CYAN)$(LIKWID)$(COLOR_NO)"))
	303	$(info $(shell $(ECHO_E) "# VERIFICATION=$(COLOR_CYAN)$(VERIFICATION)$(COLOR_NO) STATISTICS=$(COLOR_CYAN)$(STATISTICS)$(COLOR_NO) VTK_OUTPUT=$(COLOR_CYAN)$(VTK_OUTPUT)$(COLOR_NO)"))
	304	$(info $(shell $(ECHO_E) "# target=$(.DEFAULT_GOAL)"))
10988083 MW	305
	306	$(info # Object dir: $(OBJECT_DIR))
	307	$(info # Dependency dir: $(DEP_DIR))
	308	# $(info # objects: $(OBJ))
	309
	310	BIN_DIR=../bin
	311
0fde6e45	312	all: $(BIN_DIR)/lbmbenchk-$(CONFIG)-$(BUILD)$(BUILD_CONFIG)$(PREC)$(TAG)
10988083 MW	313
	314
	315	# ------------------------------------------------------------------------
	316	# TODO: Include dependencies, if they exist and no clean target is specified.
	317
	318	#ifeq (,$(SKIP_DEP_INCLUDE))
	319	# ifneq (,$(OBJ))
	320	# TMP_OBJ__ = $(patsubst $(OBJECT_DIR)/%,$(DEP_DIR)/%,$(OBJ))
	321	# $(warning TMP_OBJ__: $(TMP_OBJ__))
	322	# $(warning OBJ: $(OBJ))
	323	# $(warning DEPS: $(patsubst %.o,%.d,$(TMP_OBJ__)))
	324	# $(warning $(patsubst $(OBJECT_DIR)/%.o,$(DEP_DIR)/%.d,$(OBJ)))
	325	-include $(patsubst $(OBJECT_DIR)/%.o,$(DEP_DIR)/%.d,$(OBJ))
	326	# endif
	327	#endif
	328
	329
	330	$(BIN_DIR):
	331	[ -d "$@" ] \|\| mkdir -p "$@"
	332
0fde6e45	333	$(BIN_DIR)/lbmbenchk-$(CONFIG)-$(BUILD)$(BUILD_CONFIG)$(PREC)$(TAG): $(OBJ) $(REBUILD_DEPS) $(DEP_DIR)/.target \| $(BIN_DIR)
10988083 MW	334	@$(ECHO_E) "linking: $(COLOR_CYAN)$@$(COLOR_NO)"
	335	$(LD) $(LD_FLAGS) -o $@ $(filter-out $(REBUILD_DEPS),$^) $(LD_LIBS)
	336	@$(ECHO_E) "# Builded binary: $(COLOR_CYAN)$@$(COLOR_NO)"
0fde6e45 MW	337	@$(ECHO_E) "# Configuration was: CONFIG=$(COLOR_CYAN)$(CONFIG)$(COLOR_NO) BUILD=$(COLOR_CYAN)$(BUILD)$(COLOR_NO) PRECISION=$(COLOR_CYAN)$(PRECISION)$(COLOR_NO)"
	338	@$(ECHO_E) "# OPENMP=$(COLOR_CYAN)$(OPENMP)$(COLOR_NO) ISA=$(COLOR_CYAN)$(ISA)$(COLOR_NO) LIKWID=$(COLOR_CYAN)$(LIKWID)$(COLOR_NO)"
	339	@$(ECHO_E) "# VERIFICATION=$(COLOR_CYAN)$(VERIFICATION)$(COLOR_NO) STATISTICS=$(COLOR_CYAN)$(STATISTICS)$(COLOR_NO) VTK_OUTPUT=$(COLOR_CYAN)$(VTK_OUTPUT)$(COLOR_NO)"
	340	@$(ECHO_E) "# target=$(.DEFAULT_GOAL)"
10988083 MW	341
	342	$(OBJECT_DIR)/%_SoA.o: %.c $(REBUILD_DEPS)
	343	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO) $(COLOR_MAGENTA)DATA_LAYOUT_SOA$(COLOR_NO)"
	344	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_SOA -c $< -o $@
	345
	346	$(OBJECT_DIR)/%_PushSoA.o: %.c $(REBUILD_DEPS)
	347	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO) $(COLOR_MAGENTA)DATA_LAYOUT_SOA$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_PUSH$(COLOR_NO)"
	348	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_SOA $(D)PROP_MODEL_PUSH -c $< -o $@
	349
	350	$(OBJECT_DIR)/%_PushAoS.o: %.c $(REBUILD_DEPS)
	351	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO) $(COLOR_MAGENTA)DATA_LAYOUT_AOS$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_PUSH$(COLOR_NO)"
	352	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_AOS $(D)PROP_MODEL_PUSH -c $< -o $@
	353
8cafd9ea MW	354	$(OBJECT_DIR)/%_PushAoSoA.o: %.c $(REBUILD_DEPS)
	355	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO) $(COLOR_MAGENTA)DATA_LAYOUT_AOSOA$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_PUSH$(COLOR_NO)"
	356	$(Q)$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_AOSOA $(D)PROP_MODEL_PUSH -c $< -o $@
	357
10988083 MW	358	$(OBJECT_DIR)/%_PullSoA.o: %.c $(REBUILD_DEPS)
	359	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO) $(COLOR_MAGENTA)DATA_LAYOUT_SOA$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_PULL$(COLOR_NO)"
	360	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_SOA $(D)PROP_MODEL_PULL -c $< -o $@
	361
	362	$(OBJECT_DIR)/%_PullAoS.o: %.c $(REBUILD_DEPS)
	363	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO) $(COLOR_MAGENTA)DATA_LAYOUT_AOS$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_PULL$(COLOR_NO)"
	364	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_AOS $(D)PROP_MODEL_PULL -c $< -o $@
	365
	366	$(OBJECT_DIR)/%_AoS.o: %.c $(REBUILD_DEPS)
	367	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO) $(COLOR_MAGENTA)DATA_LAYOUT_AOS$(COLOR_NO)"
	368	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_AOS -c $< -o $@
	369
e3f82424 MW	370	$(OBJECT_DIR)/%_AaAoS.o: %.c $(REBUILD_DEPS)
	371	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO) $(COLOR_MAGENTA)DATA_LAYOUT_AOS$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_AA$(COLOR_NO)"
	372	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_AOS $(D)PROP_MODEL_AA -c $< -o $@
	373
	374	$(OBJECT_DIR)/%_AaSoA.o: %.c $(REBUILD_DEPS)
	375	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO) $(COLOR_MAGENTA)DATA_LAYOUT_SOA$(COLOR_NO) $(COLOR_MAGENTA)PROP_MODEL_AA$(COLOR_NO)"
	376	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) $(D)DATA_LAYOUT_SOA $(D)PROP_MODEL_AA -c $< -o $@
10988083 MW	377
	378	$(OBJECT_DIR)/%.o: %.c $(REBUILD_DEPS)
	379	@$(ECHO_E) "compiling: $(COLOR_CYAN)$@$(COLOR_NO)"
	380	$(CC) $(strip $(C_FLAGS)) $(strip $(PP_FLAGS)) -c $< -o $@
	381
	382
	383	# ------------------------------------------------------------------------
8cafd9ea	384	# Following targets in principal make only sure, that the directories
10988083 MW	385	# are created if the do not exist already.
	386
	387	$(DEP_DIR)/.target:
	388	$(Q)$(MKDIR_P) $(DEP_DIR)
	389	$(Q)$(TOUCH) $@
	390
	391	# ------------------------------------------------------------------------
	392	# Current configuration.
0fde6e45	393	MAKE_CFG = SYSTEM=$(SYSTEM) // BUILD=$(BUILD) // MAKEOVERRIDES=\"$(strip $(MAKEOVERRIDES))\" // VERIFICATION=$(VERIFICATION) // STATISTICS=$(STATISTICS) // VTK_OUTPUT=$(VTK_OUTPUT) // VTK_OUTPUT_ASCII=$(VTK_OUTPUT_ASCII) // LID_DRIVEN_CAVITY=$(LID_DRIVEN_CAVITY) // ISA=$(ISA) // LIKWID=$(LIKWID) // PRECISION=$(PRECISION)
10988083 MW	394
	395	# Compare current configuration to the last one so we know when to
	396	# rebuild this system/target despite when sources have not changed.
	397	$(OBJECT_DIR)/.target::
	398	$(Q)$(MKDIR_P) $(OBJECT_DIR)
	399	@ ( [ -f $(OBJECT_DIR)/.target ] && $(ECHO) $(MAKE_CFG) \| $(CMP) -s $(OBJECT_DIR)/.target - ) \
	400	\|\| ( $(ECHO) $(MAKE_CFG) > $(OBJECT_DIR)/.target; $(ECHO) "# configuration variables changed, rebuilding ..." )
	401
	402
	403	clean:
	404	-rm $(OBJECT_DIR)/.o $(DEP_DIR)/.d
	405
	406	clean-all:
	407	-[ -d obj ] && rm -r obj