.SUFFIXES:
CUDA_TRIPLE ?= x86_64-linux
CUBLAS_TRIPLE ?= x86_64-linux-gnu
DLSW_TRIPLE ?= x86_64-linux-gnu
SAFE_PDK ?= 0
TARGET ?= $(shell uname -m)

ifeq ($(CUDA_INSTALL_DIR),)
  CUDA_INSTALL_DIR ?= /usr/local/cuda
  $(warning CUDA_INSTALL_DIR variable is not specified, using $(CUDA_INSTALL_DIR) by default, use CUDA_INSTALL_DIR=<cuda_directory> to change.)
endif

ifeq ($(CUDNN_INSTALL_DIR),)
  CUDNN_INSTALL_DIR ?= $(CUDA_INSTALL_DIR)
  $(warning CUDNN_INSTALL_DIR variable is not specified, using $(CUDA_INSTALL_DIR) by default, use CUDNN_INSTALL_DIR=<cudnn_directory> to change.)
endif

ifeq ($(ENABLE_DLA), 1)
  ifeq ($(PDK_DIR),)
    $(error DLA is enabled and PDK_DIR is not specified but it is required by some samples, use PDK_DIR=<pdk_directory> to specify.)
  endif
  PDK_LIB_DIR := $(PDK_DIR)/lib-target
  PDK_INC_DIR := $(PDK_DIR)/include
endif

ifeq ($(TRT_LIB_DIR),)
  TRT_LIB_DIR ?= ../../lib
  $(warning TRT_LIB_DIR is not specified, searching $(TRT_LIB_DIR), ../../lib, ../lib by default, use TRT_LIB_DIR=<trt_lib_directory> to change.)
endif

CUDA_LIBDIR = lib
CUDNN_LIBDIR = lib64
ifeq ($(TARGET), aarch64)
  ifeq ($(shell uname -m), aarch64)
    CUDA_LIBDIR = lib64
    CC = g++
  else
    CC = aarch64-linux-gnu-g++
  endif
  CUCC = $(CUDA_INSTALL_DIR)/bin/nvcc -m64 -ccbin $(CC)
else ifeq ($(TARGET), x86_64)
  CUDA_LIBDIR = lib64
  CC = g++
  CUCC = $(CUDA_INSTALL_DIR)/bin/nvcc -m64
else ifeq ($(TARGET), ppc64le)
  CUDA_LIBDIR = lib64
  CC = g++
  CUCC = $(CUDA_INSTALL_DIR)/bin/nvcc -m64
else ifeq ($(TARGET), qnx)
  CC = ${QNX_HOST}/usr/bin/aarch64-unknown-nto-qnx7.0.0-g++
  CUCC = $(CUDA_INSTALL_DIR)/bin/nvcc -m64 -ccbin $(CC)
else ifeq ($(TARGET), android64)
  ifeq ($(ANDROID_CC),)
    $(error ANDROID_CC must be set to the clang compiler to build for android 64bit, for example /path/to/my-toolchain/bin/aarch64-linux-android-clang++)
  endif
  CUDA_LIBDIR = lib
  ANDROID_FLAGS = -DANDROID -D_GLIBCXX_USE_C99=1 -Wno-sign-compare -D__aarch64__ -Wno-strict-aliasing -Werror -pie -fPIE -Wno-unused-command-line-argument
  COMMON_FLAGS += $(ANDROID_FLAGS)
  COMMON_LD_FLAGS += $(ANDROID_FLAGS)
  CC = $(ANDROID_CC)
  CUCC = $(CUDA_INSTALL_DIR)/bin/nvcc -m64 -ccbin $(CC) --compiler-options="-DANDROID -D_GLIBCXX_USE_C99=1 -Wno-sign-compare"
  ANDROID = 1
else ########
  $(error Auto-detection of platform failed. Please specify one of the following arguments to make: TARGET=[aarch64|x86_64|qnx|android64])
endif

ifdef VERBOSE
  AT=
else
  AT=@
endif

AR = ar cr
ECHO = @echo

SHELL = /bin/sh

ROOT_PATH = ../..
ifeq ($(SAFE_PDK), 1)
  OUT_PATH = $(ROOT_PATH)/bin/safety
else
  OUT_PATH = $(ROOT_PATH)/bin
endif
OUTDIR = $(OUT_PATH)

define concat
$1$2$3$4$5$6$7$8
endef

ifneq ($(USE_QCC),1)
# Usage: $(call make-depend,source-file,object-file,depend-file)
define make-depend
  $(AT)$(CC) -MM -MF $3 -MP -MT $2 $(COMMON_FLAGS) $1
endef
# Usage: $(call make-cuda-depend,source-file,object-file,depend-file,flags)
define make-cuda-depend
  $(AT)$(CUCC) -M -MT $2 $4 $1 > $3
endef
endif

# When TRT_STATIC is set, pick the static libraries for all components. Samples are compiled with static libraries
ifeq ($(TRT_STATIC), 1)
  CUDART_LIB = -lcudart_static
  CUDNN_LIB = -lcudnn_static
  CUBLAS_LIB = -lcublas_static
  MYELIN_LIB = -lmyelin_compiler_static -lmyelin_executor_static -lmyelin_pattern_library_static -lmyelin_pattern_runtime_static
  NVINFER_LIB = -lnvinfer_static
  NVPARSERS_LIB = -lnvparsers_static
  NVINFER_PLUGIN_LIB = -lnvinfer_plugin_static
  NVONNXPARSERS_LIB = -lnvonnxparser_static
  NVRTC_LIB = -lnvrtc_static
  PROTO_LIB = -lprotobuf -lonnx_proto
  STUBS_DIR = -L"$(TRT_LIB_DIR)/stubs" -Wl,-rpath-link="$(TRT_LIB_DIR)/stubs"
else
  CUDART_LIB = -lcudart
  CUDNN_LIB = -lcudnn
  CUBLAS_LIB = -lcublas
  MYELIN_LIB = -lmyelin
  NVINFER_LIB = -lnvinfer
  NVPARSERS_LIB = -lnvparsers
  NVINFER_PLUGIN_LIB = -lnvinfer_plugin
  NVONNXPARSERS_LIB = -lnvonnxparser
  NVRTC_LIB = -lnvrtc
  PROTO_LIBDIR =
  STUBS_DIR =
endif

#########################
INCPATHS=
LIBPATHS=
COMMON_LIBS=

# Add extra libraries if TRT_STATIC is enabled
ifeq ($(TRT_STATIC), 1)
  COMMON_LIBS += -lculibos -lcublasLt_static
endif

# add cross compile directories
ifneq ($(shell uname -m), $(TARGET))
  INCPATHS += -I"/usr/include/$(DLSW_TRIPLE)" -I"/usr/include/$(CUBLAS_TRIPLE)"
  LIBPATHS += -L"../lib/stubs" -L"../../lib/stubs" -L"/usr/lib/$(DLSW_TRIPLE)/stubs" -L"/usr/lib/$(DLSW_TRIPLE)" -L"/usr/lib/$(CUBLAS_TRIPLE)/stubs" -L"/usr/lib/$(CUBLAS_TRIPLE)"
  LIBPATHS += -L"$(CUDA_INSTALL_DIR)/targets/$(CUDA_TRIPLE)/$(CUDA_LIBDIR)/stubs" -L"$(CUDA_INSTALL_DIR)/targets/$(CUDA_TRIPLE)/$(CUDA_LIBDIR)"
endif
INCPATHS += -I"../common" -I"$(CUDA_INSTALL_DIR)/include" -I"$(CUDNN_INSTALL_DIR)/include" -I"../include" -I"../../include" -I"../../parsers/onnxOpenSource"
LIBPATHS += -L"$(CUDA_INSTALL_DIR)/$(CUDA_LIBDIR)" -Wl,-rpath-link="$(CUDA_INSTALL_DIR)/$(CUDA_LIBDIR)"
LIBPATHS += -L"$(CUDNN_INSTALL_DIR)/$(CUDNN_LIBDIR)" -Wl,-rpath-link="$(CUDNN_INSTALL_DIR)/$(CUDNN_LIBDIR)"
LIBPATHS += -L"../lib" -L"../../lib" -L"$(TRT_LIB_DIR)" -Wl,-rpath-link="$(TRT_LIB_DIR)" $(STUBS_DIR)

# libnvinfer_safe.so links to neither standard nor safe PDK, while libnvinfer.so depends on standard PDK when DLA is enabled.
ifeq ($(SAFE_PDK), 0)
  ifneq ($(ENABLE_DLA), 0)
    LIBPATHS += -L"$(PDK_LIB_DIR)" -Wl,-rpath-link="$(PDK_LIB_DIR)"
  endif
endif

# delimiter ; is to avoid the issue caused by the case that one keyword is the substr of another keyword
USE_PDK_LISTS := dla_safety_runtime; sample_nvmedia;

# add required PDK headers/libraries
ifeq ($(ENABLE_DLA), 1)
  ifeq ($(TARGET), qnx)
    LIBPATHS += -L"$(QNX_TARGET)/aarch64le/lib"
    LIBPATHS += -L"$(QNX_TARGET)/aarch64le/lib/gcc/5.4.0"
    LIBPATHS += -L"$(QNX_TARGET)/aarch64le/usr/lib"
    INCPATHS += -I"$(QNX_TARGET)/usr/include"
  endif

  PDK_LISTS_FILTER := $(OUTNAME_RELEASE);
  ifneq ($(findstring $(PDK_LISTS_FILTER),$(USE_PDK_LISTS)),)
    HAS_NVSCIBUF_LIB=$(shell ls ${PDK_LIB_DIR}/libnvscibuf.so 2> /dev/null | wc -l)
    ifeq ($(HAS_NVSCIBUF_LIB), 1)
        LIBLIST += -lnvscibuf
    endif
    HAS_NVMEDIA_LIB=$(shell ls ${PDK_LIB_DIR}/libnvmedia.so 2> /dev/null | wc -l)
    ifeq ($(HAS_NVMEDIA_LIB), 1)
        LIBLIST += -lnvmedia
    endif
    HAS_NVMEDIA_CORE_LIB=$(shell ls ${PDK_LIB_DIR}/libnvmedia_core.so 2> /dev/null | wc -l)
    ifeq ($(HAS_NVMEDIA_CORE_LIB), 1)
        LIBLIST += -lnvmedia_core
    endif
    HAS_NVMEDIA_TENSOR_LIB=$(shell ls ${PDK_LIB_DIR}/libnvmedia_tensor.so 2> /dev/null | wc -l)
    ifeq ($(HAS_NVMEDIA_TENSOR_LIB), 1)
        LIBLIST += -lnvmedia_tensor
    endif
    HAS_NVMEDIA_DLA_LIB=$(shell ls ${PDK_LIB_DIR}/libnvmedia_dla.so 2> /dev/null | wc -l)
    ifeq ($(HAS_NVMEDIA_DLA_LIB), 1)
        LIBLIST += -lnvmedia_dla
    endif
    COMMON_LIBS += ${LIBLIST}
    LIBPATHS += -L"$(PDK_LIB_DIR)" -Wl,-rpath-link=$(PDK_LIB_DIR) -Wl,--unresolved-symbols=ignore-in-shared-libs
    INCPATHS += -I"$(PDK_INC_DIR)"
  endif
endif

# Add myelin libraries if applicable
ENABLE_MYELIN := 0
ifeq ($(TARGET), x86_64)
    ENABLE_MYELIN = 1
else ifeq ($(TARGET), ppc64le)
  ifeq ($(CUDA), $(filter $(CUDA), cuda-10.2))
    ENABLE_MYELIN = 1
  endif
else ifeq ($(TARGET), aarch64)
  ifeq ($(CUDA), $(filter $(CUDA), cuda-10.2))
    ENABLE_MYELIN = 1
  endif
endif
ifeq ($(ENABLE_MYELIN), 1)
  COMMON_LIBS += $(MYELIN_LIB) $(NVRTC_LIB)
endif

.SUFFIXES:
vpath %.h $(EXTRA_DIRECTORIES)
vpath %.cpp $(EXTRA_DIRECTORIES)

COMMON_FLAGS += -Wall -Wno-deprecated-declarations -std=c++11 $(INCPATHS)
ifneq ($(ANDROID),1)
  COMMON_FLAGS += -D_REENTRANT
endif

ifeq ($(TARGET), qnx)
  COMMON_FLAGS += -D_POSIX_C_SOURCE=200112L -D_QNX_SOURCE -D_FILE_OFFSET_BITS=64 -fpermissive
endif

COMMON_LD_FLAGS += $(LIBPATHS) -L$(OUTDIR)

OBJDIR = $(call concat,$(OUTDIR),/chobj)
DOBJDIR = $(call concat,$(OUTDIR),/dchobj)

COMMON_LIBS += $(CUDART_LIB)
ifneq ($(SAFE_PDK),1)
  COMMON_LIBS += $(CUBLAS_LIB) $(CUDNN_LIB)
endif

ifneq ($(TARGET), qnx)
  ifneq ($(ANDROID), 1)
    COMMON_LIBS += -lrt -ldl -lpthread
  endif
endif
ifeq ($(ANDROID),1)
  COMMON_LIBS += -lculibos -llog
endif

COMMON_LIBS_FOR_EXECUTABLE := $(filter-out -lcudart_static ,$(COMMON_LIBS))
ifeq ($(USE_CUDART_STATIC), 1)
	COMMON_LIBS_FOR_EXECUTABLE += $(CUDART_LIB)
endif

ifeq ($(SAFE_PDK),1)
  LIBS = $(COMMON_LIBS_FOR_EXECUTABLE) $(PROTO_LIB)
  DLIBS = $(COMMON_LIBS_FOR_EXECUTABLE) $(PROTO_LIB)
else
  LIBS = $(NVINFER_LIB) $(NVPARSERS_LIB) $(NVINFER_PLUGIN_LIB) $(NVONNXPARSERS_LIB) $(COMMON_LIBS_FOR_EXECUTABLE) $(PROTO_LIB)
  DLIBS = $(NVINFER_LIB) $(NVPARSERS_LIB) $(NVINFER_PLUGIN_LIB) $(NVONNXPARSERS_LIB) $(COMMON_LIBS_FOR_EXECUTABLE) $(PROTO_LIB)
endif

OBJS = $(patsubst %.cpp, $(OBJDIR)/%.o, $(wildcard *.cpp $(addsuffix /*.cpp, $(EXTRA_DIRECTORIES))))
DOBJS = $(patsubst %.cpp, $(DOBJDIR)/%.o, $(wildcard *.cpp $(addsuffix  /*.cpp, $(EXTRA_DIRECTORIES))))
CUOBJS = $(patsubst %.cu, $(OBJDIR)/%.o, $(wildcard *.cu $(addsuffix  /*.cu, $(EXTRA_DIRECTORIES))))
CUDOBJS = $(patsubst %.cu, $(DOBJDIR)/%.o, $(wildcard *.cu $(addsuffix  /*.cu, $(EXTRA_DIRECTORIES))))

CFLAGS = $(COMMON_FLAGS)
CFLAGSD = $(COMMON_FLAGS) -g
LFLAGS = $(COMMON_LD_FLAGS)
LFLAGSD = $(COMMON_LD_FLAGS)

all: debug release

release : $(OUTDIR)/$(OUTNAME_RELEASE)

debug   : $(OUTDIR)/$(OUTNAME_DEBUG)

test: test_debug test_release

test_debug:
	$(AT)cd $(OUTDIR) && ./$(OUTNAME_DEBUG)

test_release:
	$(AT)cd $(OUTDIR) && ./$(OUTNAME_RELEASE)

ifdef MAC
$(OUTDIR)/$(OUTNAME_RELEASE) : $(OBJS) $(CUOBJS)
	$(ECHO) Linking: $@
	$(AT)$(CC) -o $@ $^ $(LFLAGS) $(LIBS)
	# Copy every EXTRA_FILE of this sample to bin dir
	$(foreach EXTRA_FILE,$(EXTRA_FILES), cp -f $(EXTRA_FILE) $(OUTDIR)/$(EXTRA_FILE); )

$(OUTDIR)/$(OUTNAME_DEBUG) : $(DOBJS) $(CUDOBJS)
	$(ECHO) Linking: $@
	$(AT)$(CC) -o $@ $^ $(LFLAGSD) $(DLIBS)
else
$(OUTDIR)/$(OUTNAME_RELEASE) : $(OBJS) $(CUOBJS)
	$(ECHO) Linking: $@
	$(AT)$(CC) -o $@ $^ $(LFLAGS) -Wl,--start-group $(LIBS) -Wl,--end-group
	# Copy every EXTRA_FILE of this sample to bin dir
	$(foreach EXTRA_FILE,$(EXTRA_FILES), cp -f $(EXTRA_FILE) $(OUTDIR)/$(EXTRA_FILE); )

$(OUTDIR)/$(OUTNAME_DEBUG) : $(DOBJS) $(CUDOBJS)
	$(ECHO) Linking: $@
	$(AT)$(CC) -o $@ $^ $(LFLAGSD) -Wl,--start-group $(DLIBS) -Wl,--end-group
endif

$(OBJDIR)/%.o: %.cpp
	$(AT)if [ ! -d $(OBJDIR) ]; then mkdir -p $(OBJDIR); fi
	$(foreach XDIR,$(EXTRA_DIRECTORIES), if [ ! -d $(OBJDIR)/$(XDIR) ]; then mkdir -p $(OBJDIR)/$(XDIR); fi;) :
	$(call make-depend,$<,$@,$(subst .o,.d,$@))
	$(ECHO) Compiling: $<
	$(AT)$(CC) $(CFLAGS) -c -o $@ $<

$(DOBJDIR)/%.o: %.cpp
	$(AT)if [ ! -d $(DOBJDIR) ]; then mkdir -p $(DOBJDIR); fi
	$(foreach XDIR,$(EXTRA_DIRECTORIES), if [ ! -d $(OBJDIR)/$(XDIR) ]; then mkdir -p $(DOBJDIR)/$(XDIR); fi;) :
	$(call make-depend,$<,$@,$(subst .o,.d,$@))
	$(ECHO) Compiling: $<
	$(AT)$(CC) $(CFLAGSD) -c -o $@ $<

######################################################################### CU
$(OBJDIR)/%.o: %.cu
	$(AT)if [ ! -d $(OBJDIR) ]; then mkdir -p $(OBJDIR); fi
	$(foreach XDIR,$(EXTRA_DIRECTORIES), if [ ! -d $(OBJDIR)/$(XDIR) ]; then mkdir -p $(OBJDIR)/$(XDIR); fi;) :
	$(call make-cuda-depend,$<,$@,$(subst .o,.d,$@))
	$(ECHO) Compiling CUDA release: $<
	$(AT)$(CUCC) $(CUFLAGS) -c -o $@ $<

$(DOBJDIR)/%.o: %.cu
	$(AT)if [ ! -d $(DOBJDIR) ]; then mkdir -p $(DOBJDIR); fi
	$(foreach XDIR,$(EXTRA_DIRECTORIES), if [ ! -d $(DOBJDIR)/$(XDIR) ]; then mkdir -p $(DOBJDIR)/$(XDIR); fi;) :
	$(call make-cuda-depend,$<,$@,$(subst .o,.d,$@))
	$(ECHO) Compiling CUDA debug: $<
	$(AT)$(CUCC) $(CUFLAGSD) -c -o $@ $<

clean:
	$(ECHO) Cleaning...
	$(foreach XDIR,$(EXTRA_DIRECTORIES), if [ -d $(OBJDIR)/$(XDIR) ]; then rm -rf $(OBJDIR)/$(XDIR); fi;) :
	$(foreach XDIR,$(EXTRA_DIRECTORIES), if [ -d $(DOBJDIR)/$(XDIR) ]; then rm -rf $(DOBJDIR)/$(XDIR); fi;) :
	$(AT)rm -rf $(OBJDIR) $(DOBJDIR) $(OUTDIR)/$(OUTNAME_RELEASE) $(OUTDIR)/$(OUTNAME_DEBUG)
	$(foreach EXTRA_FILE,$(EXTRA_FILES), if [ -f $(OUTDIR)/$(EXTRA_FILE)  ]; then rm -f $(OUTDIR)/$(EXTRA_FILE); fi;) :

ifneq "$(MAKECMDGOALS)" "clean"
	-include $(OBJDIR)/*.d $(DOBJDIR)/*.d
endif # ifneq "$(MAKECMDGOALS)" "clean"