# SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: LicenseRef-NvidiaProprietary
#
# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
# property and proprietary rights in and to this material, related
# documentation and any modifications thereto. Any use, reproduction,
# disclosure or distribution of this material and related documentation
# without an express license agreement from NVIDIA CORPORATION or
# its affiliates is strictly prohibited.

.SUFFIXES:
CUDA_TRIPLE ?= x86_64-linux
DLSW_TRIPLE ?= x86_64-linux-gnu
PROTOBUF_TRIPLE ?= x86_64-linux-gnu
SAFE_PDK ?= 0
CPP_STANDARD ?= 14
TARGET ?= $(shell uname -m)
CUSTOM_LD_FLAGS ?=
USE_STUB_EXTERNALS ?= 0
TRT_STATIC ?= 0
GENCODES ?=

ifeq ($(CUDA_INSTALL_DIR), )
  CUDA_INSTALL_DIR ?= /usr/local/cuda
  $(warning CUDA_INSTALL_DIR variable is not specified, using $(CUDA_INSTALL_DIR) by default, use CUDA_INSTALL_DIR=<cuda_directory> to change.)
endif

ifeq ($(ENABLE_DLA), 1)
  ifneq ($(PDK_DIR), )
    PDK_LIB_DIR ?= $(PDK_DIR)/lib-target
    PDK_INC_DIR ?= $(PDK_DIR)/include
  endif
  ifeq ($(PDK_LIB_DIR), )
    PDK_LIB_DIR ?= /usr/lib
    $(warning DLA is enabled and PDK_LIB_DIR is not specified, using $(PDK_LIB_DIR) by default, use PDK_LIB_DIR=<pdk_directory> to change.)
  endif
  ifeq ($(PDK_INC_DIR), )
    PDK_INC_DIR ?= /usr/include
    $(warning DLA is enabled and PDK_INC_DIR is not specified, using $(PDK_INC_DIR) by default, use PDK_INC_DIR=<pdk_directory> to change.)
  endif
endif

ifeq ($(TRT_LIB_DIR), )
  TRT_LIB_DIR ?= ../../lib
  $(warning TRT_LIB_DIR is not specified, searching $(TRT_LIB_DIR), ../../lib, ../lib by default, use TRT_LIB_DIR=<trt_lib_directory> to change.)
endif

ifeq ($(PROTOBUF_INSTALL_DIR), )
  PROTOBUF_INSTALL_DIR ?= /usr/lib/$(PROTOBUF_TRIPLE)
  $(warning PROTOBUF_INSTALL_DIR variable is not specified, use PROTOBUF_INSTALL_DIR=<protobuf_directory> to change. Using default value: $(PROTOBUF_INSTALL_DIR))
endif

CUDA_LIBDIR = lib
ifeq ($(TARGET), aarch64)
  ifeq ($(shell uname -m), aarch64)
    CUDA_LIBDIR = lib64
    CC = g++
  else
    CC = aarch64-linux-gnu-g++
  endif
  CUCC = $(CUDA_INSTALL_DIR)/bin/nvcc -m64 -ccbin $(CC)
else ifeq ($(TARGET), x86_64)
  CUDA_LIBDIR = lib64
  CC = g++
  CUCC = $(CUDA_INSTALL_DIR)/bin/nvcc -m64
else ifeq ($(TARGET), ppc64le)
  CUDA_LIBDIR = lib64
  CC = g++
  CUCC = $(CUDA_INSTALL_DIR)/bin/nvcc -m64
else ifeq ($(TARGET), qnx)
  ifndef QNX_VERSION
    $(error QNX_VERSION is not set)
  endif
  CC = ${QNX_HOST}/usr/bin/aarch64-unknown-nto-qnx$(QNX_VERSION)-g++
  ifeq ($(USE_QCC), 1)
    CC = ${QNX_HOST}/usr/bin/q++
    COMMON_FLAGS += -D_QNX_SOURCE -D__aarch64__ -V8.3.0,gcc_ntoaarch64le
    COMMON_LD_FLAGS += -D_QNX_SOURCE -D__aarch64__ -V8.3.0,gcc_ntoaarch64le
  endif
  CUCC = $(CUDA_INSTALL_DIR)/bin/nvcc -m64 -ccbin $(CC)
else ########
  $(error Auto-detection of platform failed. Please specify one of the following arguments to make: TARGET=[aarch64|x86_64|qnx])
endif

CC_MAJOR_VER = $(shell $(CC) -dumpversion | sed -e "s/\..*//")
ifeq ($(CC_MAJOR_VER), 4)
  CPP_STANDARD := 11
endif

# When using GCC_MAJOR_VERSION < 8 on x86_64 Linux, RedHat Developer Toolset 8 non-shared libstdc++ library is needed
# to static link TensorRT.
# For more details, see https://docs.nvidia.com/deeplearning/tensorrt/sample-support-guide/index.html#building-samples-limitations.
ifeq ($(TRT_STATIC), 1)
  ifeq ($(TARGET), x86_64)
    ifeq ($(shell expr $(CC_MAJOR_VER) \< 8), 1)
      ifeq ($(REDHAT_DEVTOOL_LIB_DIR), )
        REDHAT_DEVTOOL_LIB_DIR ?= $(TRT_LIB_DIR)
        $(warning REDHAT_DEVTOOL_LIB_DIR is not specified, searching $(TRT_LIB_DIR), ../../lib, ../lib by default, use REDHAT_DEVTOOL_LIB_DIR=<redhat_devtool_lib_directory> to change.)
      endif

      REDHAT_DEVTOOL_LIB = $(shell test -e "$(REDHAT_DEVTOOL_LIB_DIR)/libstdc++_nonshared.a" && echo -lstdc++_nonshared)
      ifeq ($(REDHAT_DEVTOOL_LIB), )
        REDHAT_DEVTOOL_LIB = $(shell test -e "../lib/libstdc++_nonshared.a" && echo -lstdc++_nonshared)
      endif
      ifeq ($(REDHAT_DEVTOOL_LIB), )
        REDHAT_DEVTOOL_LIB = $(shell test -e "../../lib/libstdc++_nonshared.a" && echo -lstdc++_nonshared)
      endif
      ifeq ($(REDHAT_DEVTOOL_LIB), )
        $(warning libstdc++_nonshared.a cannot be found in $(REDHAT_DEVTOOL_LIB_DIR), ../../lib, ../lib. Trying to compile without libstdc++_nonshared.a. \
          For more details, see https://docs.nvidia.com/deeplearning/tensorrt/sample-support-guide/index.html#building-samples-limitations.)
      endif
    endif
  endif
endif

ifdef VERBOSE
  AT =
else
  AT = @
endif

AR = ar cr
ECHO = @echo

SHELL = /bin/sh

ROOT_PATH = ../..
ifeq ($(SAFE_PDK), 1)
  OUT_PATH = $(ROOT_PATH)/bin/safety
else
  OUT_PATH = $(ROOT_PATH)/bin
endif
OUTDIR = $(OUT_PATH)

define concat
$1$2$3$4$5$6$7$8
endef

ifneq ($(USE_QCC), 1)
# Usage: $(call make-depend,source-file,object-file,depend-file)
define make-depend
  $(AT)$(CC) -MM -MF $3 -MP -MT $2 $(COMMON_FLAGS) $1
endef
# Usage: $(call make-cuda-depend,source-file,object-file,depend-file,flags)
define make-cuda-depend
  $(AT)$(CUCC) -M -MT $2 $(CUFLAGS) $4 $1 > $3
endef
endif

USE_NVRTC_STATIC = 0
ifneq ("$(wildcard $(CUDA_INSTALL_DIR)/targets/$(CUDA_TRIPLE)/lib*/libnvrtc_static.a)", "")
  USE_NVRTC_STATIC = 1
endif
USE_PTXJIT_STATIC = 0
ifneq ("$(wildcard $(CUDA_INSTALL_DIR)/targets/$(CUDA_TRIPLE)/lib*/libnvptxcompiler_static.a)", "")
  USE_PTXJIT_STATIC = 1
endif


# When TRT_STATIC is set, pick the static libraries for all components. Samples are compiled with static libraries
ifeq ($(TRT_STATIC), 1)
  USE_CUDART_STATIC = 1
  CUDA_LIBS = -lcudart_static
  ifeq ($(USE_NVRTC_STATIC), 1)
    CUDA_LIBS += -lnvrtc_static -lnvrtc-builtins_static
  endif
  ifeq ($(USE_PTXJIT_STATIC), 1)
    CUDA_LIBS += -lnvptxcompiler_static
  endif
  NVINFER_LIB = -Wl,--whole-archive -lnvinfer_static -Wl,--no-whole-archive
  ifeq ($(TRT_WINJIT), 1)
    NVINFER_PLUGIN_LIB =
  else
    NVINFER_PLUGIN_LIB = -Wl,--whole-archive -lnvinfer_plugin_static -Wl,--no-whole-archive
  endif
  NVONNXPARSERS_LIB = -Wl,--whole-archive -lnvonnxparser_static -Wl,--no-whole-archive
  PROTO_LIB = -lonnx_proto
  STUBS_DIR = -L"$(TRT_LIB_DIR)/stubs" -Wl,-rpath-link="$(TRT_LIB_DIR)/stubs"
else
  CUDA_LIBS = -lcudart
  NVINFER_LIB = -lnvinfer
  ifeq ($(TRT_WINJIT), 1)
    NVINFER_PLUGIN_LIB =
  else
    NVINFER_PLUGIN_LIB = -lnvinfer_plugin
  endif
  NVONNXPARSERS_LIB = -lnvonnxparser
  PROTO_LIBDIR =
  STUBS_DIR =
endif

#########################
INCPATHS =
LIBPATHS =
COMMON_LIBS =

# Add extra libraries if TRT_STATIC is enabled
ifeq ($(TRT_STATIC), 1)
  # Increases the distance for jump instructions to allow larger binaries to be created, which is needed when all libraries are statically linked
  COMMON_LIBS += -lculibos
  COMMON_LD_FLAGS += -mcmodel=large
  ifneq ($(REDHAT_DEVTOOL_LIB), )
    COMMON_LIBS += $(REDHAT_DEVTOOL_LIB)
    LIBPATHS += -L"$(REDHAT_DEVTOOL_LIB_DIR)"
  endif
  # Add static tag to binaries
  ifneq ($(OUTNAME_RELEASE), )
    OUTNAME_RELEASE := $(OUTNAME_RELEASE)_static
  endif
  ifneq ($(OUTNAME_DEBUG), )
    OUTNAME_DEBUG := $(OUTNAME_DEBUG)_static
  endif
endif

# add cross compile directories
ifneq ($(shell uname -m), $(TARGET))
  INCPATHS += -I"/usr/include/$(DLSW_TRIPLE)" -I"$(CUDA_INSTALL_DIR)/targets/$(CUDA_TRIPLE)/include"
  LIBPATHS += -L"../lib/stubs" -L"../../lib/stubs" -L"/usr/lib/$(DLSW_TRIPLE)/stubs" -L"/usr/lib/$(DLSW_TRIPLE)"
  LIBPATHS += -L"$(CUDA_INSTALL_DIR)/targets/$(CUDA_TRIPLE)/$(CUDA_LIBDIR)/stubs" -L"$(CUDA_INSTALL_DIR)/targets/$(CUDA_TRIPLE)/$(CUDA_LIBDIR)"
endif
INCPATHS += -I"../common" -I"../utils" -I".." -I"$(CUDA_INSTALL_DIR)/include" -I"../include" -I"../../include" -I"../../parsers/onnxOpenSource"
LIBPATHS += -L"$(CUDA_INSTALL_DIR)/$(CUDA_LIBDIR)" -Wl,-rpath-link="$(CUDA_INSTALL_DIR)/$(CUDA_LIBDIR)"
LIBPATHS += -L"../lib" -L"../../lib" -L"$(TRT_LIB_DIR)" -Wl,-rpath-link="$(TRT_LIB_DIR)" $(STUBS_DIR)
LIBPATHS += -L"$(PROTOBUF_INSTALL_DIR)/lib" -Wl,-rpath-link="$(PROTOBUF_INSTALL_DIR)/lib"

# libnvinfer_safe.so links to neither standard nor safe PDK, while libnvinfer.so depends on standard PDK when DLA is enabled.
ifeq ($(SAFE_PDK), 0)
  ifneq ($(ENABLE_DLA), 0)
    LIBPATHS += -L"$(PDK_LIB_DIR)" -Wl,-rpath-link="$(PDK_LIB_DIR)"
  endif
endif

# delimiter ; is to avoid the issue caused by the case that one keyword is the substr of another keyword
USE_PDK_LISTS := dla_safety_runtime; sample_nvmedia;

# add required PDK headers/libraries
ifeq ($(ENABLE_DLA), 1)
  ifeq ($(TARGET), qnx)
    LIBPATHS += -L"$(QNX_TARGET)/aarch64le/lib"
    LIBPATHS += -L"$(QNX_TARGET)/aarch64le/lib/gcc/$(QNX_GCC_VERSION)"
    LIBPATHS += -L"$(QNX_TARGET)/aarch64le/usr/lib"
    INCPATHS += -I"$(QNX_TARGET)/usr/include"
  endif

  PDK_LISTS_FILTER := $(OUTNAME_RELEASE);

  PDK_LIBLIST :=
  HAS_NVSCIBUF_LIB = $(shell ls $(PDK_LIB_DIR)/libnvscibuf.so 2> /dev/null | wc -l)
  ifeq ($(HAS_NVSCIBUF_LIB), 1)
    PDK_LIBLIST += -lnvscibuf
  endif
  HAS_NVMEDIA_TENSOR_LIB = $(shell ls $(PDK_LIB_DIR)/libnvmedia_tensor.so 2> /dev/null | wc -l)
  ifeq ($(HAS_NVMEDIA_TENSOR_LIB), 1)
    PDK_LIBLIST += -lnvmedia_tensor
  endif
  HAS_NVMEDIA_DLA_LIB = $(shell ls $(PDK_LIB_DIR)/libnvmedia_dla.so 2> /dev/null | wc -l)
  ifeq ($(HAS_NVMEDIA_DLA_LIB), 1)
    PDK_LIBLIST += -lnvmedia_dla
  endif

  ifneq ($(findstring $(PDK_LISTS_FILTER),$(USE_PDK_LISTS)), )
    LIBLIST += $(PDK_LIBLIST)
  else ifeq ($(TRT_STATIC), 1)
    LIBLIST += $(PDK_LIBLIST)
    LIBLIST += -lEGL
    HAS_NVDLA_COMPILER_LIB = $(shell ls $(PDK_LIB_DIR)/libnvdla_compiler.so 2> /dev/null | wc -l)
    ifeq ($(HAS_NVDLA_COMPILER_LIB), 1)
      LIBLIST += -lnvdla_compiler
    endif
  endif

  LIBLIST += -lcudla

  COMMON_LIBS += $(LIBLIST)
  LIBPATHS += -L"$(PDK_LIB_DIR)" -Wl,-rpath-link=$(PDK_LIB_DIR) -Wl,--unresolved-symbols=ignore-in-shared-libs
  INCPATHS += -I"$(PDK_INC_DIR)"
endif

.SUFFIXES:
vpath %.h $(EXTRA_DIRECTORIES)
vpath %.cpp $(EXTRA_DIRECTORIES)

COMMON_FLAGS += -Wall -Wno-deprecated-declarations -std=c++$(CPP_STANDARD) $(INCPATHS)
COMMON_FLAGS += -D_REENTRANT
COMMON_FLAGS += -DTRT_STATIC=$(TRT_STATIC)

ifeq ($(TARGET), qnx)
  COMMON_FLAGS += -D_POSIX_C_SOURCE=200112L -D_QNX_SOURCE -D_FILE_OFFSET_BITS=64 -fpermissive
endif

COMMON_LD_FLAGS += $(LIBPATHS) -L$(OUTDIR)
COMMON_LD_FLAGS += $(CUSTOM_LD_FLAGS)

ifdef SAMPLE_DIR_NAME
  OBJDIR = $(call concat,$(OUTDIR),/chobj/,$(SAMPLE_DIR_NAME),/,$(SAMPLE_DIR_NAME))
  DOBJDIR = $(call concat,$(OUTDIR),/dchobj/,$(SAMPLE_DIR_NAME),/,$(SAMPLE_DIR_NAME))
else
  OBJDIR = $(call concat,$(OUTDIR),/chobj)
  DOBJDIR = $(call concat,$(OUTDIR),/dchobj)
endif

COMMON_LIBS += $(CUDA_LIBS)

ifneq ($(TARGET), qnx)
  COMMON_LIBS += -lrt -ldl -lpthread
endif

ifeq ($(TRT_STATIC), 1)
  COMMON_LIBS += -lz
endif

COMMON_LIBS_FOR_EXECUTABLE := $(filter-out -lcudart_static, $(COMMON_LIBS))
ifeq ($(USE_CUDART_STATIC), 1)
  COMMON_LIBS_FOR_EXECUTABLE += $(CUDA_LIBS)
endif

ifeq ($(SAFE_PDK), 1)
  LIBS = $(COMMON_LIBS_FOR_EXECUTABLE) $(PROTO_LIB)
  DLIBS = $(COMMON_LIBS_FOR_EXECUTABLE) $(PROTO_LIB)
else ifeq ($(OUTNAME_RELEASE), trtexec)
  LIBS = $(COMMON_LIBS_FOR_EXECUTABLE) $(PROTO_LIB)
  DLIBS = $(COMMON_LIBS_FOR_EXECUTABLE) $(PROTO_LIB)
else
  LIBS = $(NVINFER_LIB) $(NVINFER_PLUGIN_LIB) $(NVONNXPARSERS_LIB) $(COMMON_LIBS_FOR_EXECUTABLE) $(PROTO_LIB)
  DLIBS = $(NVINFER_LIB) $(NVINFER_PLUGIN_LIB) $(NVONNXPARSERS_LIB) $(COMMON_LIBS_FOR_EXECUTABLE) $(PROTO_LIB)
endif

OBJS = $(patsubst %.cpp, $(OBJDIR)/%.o, $(wildcard *.cpp $(addsuffix /*.cpp, $(EXTRA_DIRECTORIES))))
DOBJS = $(patsubst %.cpp, $(DOBJDIR)/%.o, $(wildcard *.cpp $(addsuffix /*.cpp, $(EXTRA_DIRECTORIES))))
CUOBJS = $(patsubst %.cu, $(OBJDIR)/%.o, $(wildcard *.cu $(addsuffix /*.cu, $(EXTRA_DIRECTORIES))))
CUDOBJS = $(patsubst %.cu, $(DOBJDIR)/%.o, $(wildcard *.cu $(addsuffix /*.cu, $(EXTRA_DIRECTORIES))))

CFLAGS = $(COMMON_FLAGS)
CFLAGSD = $(COMMON_FLAGS) -g
LFLAGS = $(COMMON_LD_FLAGS)
LFLAGSD = $(COMMON_LD_FLAGS)

CUFLAGS += $(GENCODES) $(INCPATHS)
CUFLAGSD = $(CUFLAGS) -g

all: debug release

release: $(OUTDIR)/$(OUTNAME_RELEASE)
debug: $(OUTDIR)/$(OUTNAME_DEBUG)

test: test_debug test_release

test_debug:
	$(AT)cd $(OUTDIR) && ./$(OUTNAME_DEBUG)

test_release:
	$(AT)cd $(OUTDIR) && ./$(OUTNAME_RELEASE)

$(OUTDIR)/$(OUTNAME_RELEASE): $(OBJS) $(CUOBJS)
	$(ECHO) Linking: $@
	$(AT)$(CC) -o $@ $(LFLAGS) -Wl,--start-group $(LIBS) $^ -Wl,--end-group -Wl,--no-relax

$(OUTDIR)/$(OUTNAME_DEBUG): $(DOBJS) $(CUDOBJS)
	$(ECHO) Linking: $@
	$(AT)$(CC) -o $@ $(LFLAGSD) -Wl,--start-group $(DLIBS) $^ -Wl,--end-group -Wl,--no-relax

$(OBJDIR)/%.o: %.cpp
	$(AT)if [ ! -d $(OBJDIR) ]; then mkdir -p $(OBJDIR); fi
	$(foreach XDIR, $(EXTRA_DIRECTORIES), if [ ! -d $(OBJDIR)/$(XDIR) ]; then mkdir -p $(OBJDIR)/$(XDIR); fi &&) :
	$(call make-depend,$<,$@,$(subst .o,.d,$@))
	$(ECHO) Compiling: $<
	$(AT)$(CC) $(CFLAGS) -c -o $@ $<

$(DOBJDIR)/%.o: %.cpp
	$(AT)if [ ! -d $(DOBJDIR) ]; then mkdir -p $(DOBJDIR); fi
	$(foreach XDIR, $(EXTRA_DIRECTORIES), if [ ! -d $(OBJDIR)/$(XDIR) ]; then mkdir -p $(DOBJDIR)/$(XDIR); fi &&) :
	$(call make-depend,$<,$@,$(subst .o,.d,$@))
	$(ECHO) Compiling: $<
	$(AT)$(CC) $(CFLAGSD) -c -o $@ $<

######################################################################### CU
$(OBJDIR)/%.o: %.cu
	$(AT)if [ ! -d $(OBJDIR) ]; then mkdir -p $(OBJDIR); fi
	$(foreach XDIR, $(EXTRA_DIRECTORIES), if [ ! -d $(OBJDIR)/$(XDIR) ]; then mkdir -p $(OBJDIR)/$(XDIR); fi &&) :
	$(call make-cuda-depend,$<,$@,$(subst .o,.d,$@))
	$(ECHO) Compiling CUDA release: $<
	$(AT)$(CUCC) $(CUFLAGS) -c -o $@ $<

$(DOBJDIR)/%.o: %.cu
	$(AT)if [ ! -d $(DOBJDIR) ]; then mkdir -p $(DOBJDIR); fi
	$(foreach XDIR, $(EXTRA_DIRECTORIES), if [ ! -d $(DOBJDIR)/$(XDIR) ]; then mkdir -p $(DOBJDIR)/$(XDIR); fi &&) :
	$(call make-cuda-depend,$<,$@,$(subst .o,.d,$@))
	$(ECHO) Compiling CUDA debug: $<
	$(AT)$(CUCC) $(CUFLAGSD) -c -o $@ $<

clean:
	$(ECHO) Cleaning...
	$(foreach XDIR, $(EXTRA_DIRECTORIES), if [ -d $(OBJDIR)/$(XDIR) ]; then rm -rf $(OBJDIR)/$(XDIR); fi &&) :
	$(foreach XDIR, $(EXTRA_DIRECTORIES), if [ -d $(DOBJDIR)/$(XDIR) ]; then rm -rf $(DOBJDIR)/$(XDIR); fi &&) :
	$(AT)rm -rf $(OBJDIR) $(DOBJDIR) $(OUTDIR)/$(OUTNAME_RELEASE) $(OUTDIR)/$(OUTNAME_DEBUG)

ifneq ($(MAKECMDGOALS), clean)
  -include $(OBJDIR)/*.d $(DOBJDIR)/*.d
endif