-
Notifications
You must be signed in to change notification settings - Fork 17
/
Makefile.cuda
131 lines (115 loc) · 3.93 KB
/
Makefile.cuda
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
HOSTNAME := $(shell hostname)
#HOSTNAME=ray
ifeq ($(findstring ray,$(HOSTNAME)),ray)
NVCC = /usr/local/cuda-8.0/bin/nvcc
HOSTCOMP = mpicxx
MPIPATH = /opt/ibm/spectrum_mpi
gpuarch := sm_60
EXTRA_LINK_FLAGS = -lmpi_ibm -L/usr/tcetmp/packages/lapack/lapack-3.6.0-gfortran-4.8.5/lib -llapack -L/usr/tcetmp/packages/blas/blas-3.6.0-gfortran-4.8.5/lib -lblas -lgfortran
debugdir := debug_cuda_ray
optdir := optimize_cuda_ray
else ifeq ($(findstring surface,$(HOSTNAME)),surface)
NVCC = nvcc
HOSTCOMP = /usr/local/tools/ic-16.0.258/bin/icpc
MPIPATH = /usr/local/tools/mvapich2-intel-2.2
gpuarch := sm_35
EXTRA_LINK_FLAGS = -lmpich -lmpl -lpmi -llapack
MPIINC = $(MPIPATH)/include
debugdir := debug_cuda_surface
optdir := optimize_cuda_surface
else ifeq ($(findstring lassen,$(HOSTNAME)),lassen)
NVCC = nvcc
HOSTCOMP = xlc++
FC = xlf90
MPIPATH = $(MPI_ROOT)
gpuarch := sm_70
EXTRA_LINK_FLAGS = -lmpi_ibm -L$(LAPACK_DIR) -llapack -lblas
MPIINC = $(MPIPATH)/include
debugdir := debug_cuda_lassen
optdir := optimize_cuda_lassen
else
NVCC = nvcc
HOSTCOMP = gcc
MPIPATH = $(dir $(shell which mpicc))../
gpuarch := sm_60
EXTRA_LINK_FLAGS = -lmpi -llapack
MPIINC = $(MPIPATH)/include
debugdir := debug_cuda
optdir := optimize_cuda
endif
CXX = $(NVCC)
MPILIB = $(MPIPATH)/lib
ifeq ($(debug),yes)
FFLAGS = -g
CXXFLAGS = -g -x cu -I../src -c -dc -arch=$(gpuarch) -DSW4_CROUTINES -DSW4_CUDA -DSW4_NONBLOCKING -ccbin $(HOSTCOMP)
CFLAGS = -g
else
FFLAGS = -O3
CXXFLAGS = -O3 -x cu -I../src -c -dc -arch=$(gpuarch) -DSW4_CROUTINES -DSW4_CUDA -DSW4_NONBLOCKING -ccbin $(HOSTCOMP) -Xptxas -v
CFLAGS = -O
endif
ifdef MPIINC
CXXFLAGS += -I$(MPIINC)
endif
#FFLAGS += -Mcuda
ifeq ($(openmp),yes)
OMPOPT = -Xcompiler -fopenmp
CXXFLAGS += $(OMPOPT) -DSW4_OPENMP
CFLAGS += $(OMPOPT) -DSW4_OPENMP
endif
ifeq ($(cuda_aware_mpi),yes)
CXXFLAGS += -DSW4_CUDA_AWARE_MPI
CFLAGS += -DSW4_CUDA_AWARE_MPI
endif
ifeq ($(openmp),yes)
debugdir := $(debugdir)_mp
optdir := $(optdir)_mp
endif
ifeq ($(single),yes)
debugdir := $(debugdir)_sp
optdir := $(optdir)_sp
CXXFLAGS += -I../src/float
else
CXXFLAGS += -I../src/double
endif
ifdef EXTRA_CXX_FLAGS
CXXFLAGS += $(EXTRA_CXX_FLAGS)
endif
ifdef EXTRA_FORT_FLAGS
FFLAGS += $(EXTRA_FORT_FLAGS)
endif
ifdef EXTRA_LINK_FLAGS
linklibs += $(EXTRA_LINK_FLAGS)
endif
ifeq ($(debug),yes)
builddir = $(debugdir)
else
builddir = $(optdir)
endif
OBJ = main.o EW.o Source.o rhs4sg.o rhs4sg_rev.o SuperGrid.o GridPointSource.o time_functions_cu.o ew-cfromfort.o EW_cuda.o Sarray.o device-routines.o EWCuda.o CheckPoint.o Parallel_IO.o EW-dg.o MaterialData.o MaterialBlock.o Polynomial.o SecondOrderSection.o TimeSeries.o sacsubc.o curvilinear-c.o rhs4sgcurv.o rhs4sgcurv_rev.o
FOBJ = $(addprefix $(builddir)/,$(OBJ))
sw4lite: $(FOBJ)
@echo "********* User configuration variables **************"
@echo "debug=" $(debug) " proj=" $(proj) " etree=" $(etree) " SW4ROOT"= $(SW4ROOT)
@echo "CXX=" $(CXX) "EXTRA_CXX_FLAGS"= $(EXTRA_CXX_FLAGS)
@echo "FC=" $(FC) " EXTRA_FORT_FLAGS=" $(EXTRA_FORT_FLAGS)
@echo "EXTRA_LINK_FLAGS"= $(EXTRA_LINK_FLAGS)
@echo "******************************************************"
cd $(builddir); $(NVCC) -L$(MPILIB) $(OMPOPT) -Xlinker "-rpath=$(MPILIB)" -arch=$(gpuarch) -ccbin $(HOSTCOMP) -o $@ $(OBJ) -lcudart $(linklibs)
@cat wave.txt
@echo "*** Build directory: " $(builddir) " ***"
#$(builddir)/device-routines.o:src/device-routines.C
# /bin/mkdir -p $(builddir)
# cd $(builddir); $(NVCC) $(CXXFLAGS) -c ../$<
$(builddir)/%.o:src/%.C
/bin/mkdir -p $(builddir)
cd $(builddir); $(CXX) $(CXXFLAGS) -c ../$<
$(builddir)/%.o:src/%.f
/bin/mkdir -p $(builddir)
cd $(builddir); $(FC) $(FFLAGS) -c ../$<
clean:
/bin/mkdir -p $(optdir)
/bin/mkdir -p $(debugdir)
cd $(optdir);/bin/rm -f *.o sw4lite
cd $(debugdir);/bin/rm -f *.o sw4lite
# rm -r optimize_cuda_ray optimize_cuda_ray_sp