CMAQ Installation Guide

## This document has been created by Taufiq Hassan||Email:taufiq@gist.ac.kr||Date: 20-03-2014||Updated: 21-03-2014
## In this guide, informations will be given after ## mark and shell commands with $ mark, as a superuser (s) mark
## Before starting make sure you have netcdf lib ($ find / -name "libnetcdf.a" (s))   and cvs ($ which cvs)
## For netcdf compilation refer to my WRF compilation script
## If you do not have these; install them and remember to use the same compiler (e.g. gfortran/pgf90 . .) [important]
## If you are planning to ultimately compile WRF-CMAQ coupled system; use pgf90 compiler (supported)
## Copy files from Taufiq@gcm2:/home/Taufiq/WRF-CMAQ
## Always check the log file with ctrl+f (keywords: error, warning, abort, failed, unbalanced, quote)
## Check at the end of every run.log for successful completion

$ export work=~/work (## your working directory)
$ cd $work
$ scp Taufiq@gcm2:/home/Taufiq/WRF-CMAQ/\{ioapi-3.1.tar,CMAQv5.tar,CMAQ_installation_guide,DATA.CMAQv5.tar,DATA_REF.CMAQv5.tar,config.cmaq\} .
$ tar -xvf ioapi-3.1.tar

## Installation of IOAPI (must be 3.1) for CMAQ CCTM
## It is important to know exact compiler and machine arch ($ uname -a)(I used pgf with gcc for Linux2_x86_64)
##===============================IOAPI Build Start============================================================
##============================================================================================================
$ cd ioapi
$ ls -al Makeinclude.*
## Choose Makeinclude file according to your arch/compiler (I chose Linux2_x86_64pg_gcc_nomp)
## In this file make the following changes
(1) CC = /usr/bin/gcc
(2) OMPFLAGS  =
(3) COPTFLAGS = -O2 -ffast-math -DIOAPI_NO_STDOUT
(4) FOPTFLAGS = -O2 -fast -DIOAPI_NO_STDOUT
(5) FSFLAGS   = -Bstatic

$ export BIN=Linux2_x86_64pg_gcc_nomp
$ cp Makefile.nocpl Makefile
$ vi Makefile
## Make the follwoing changes
(1) BASEDIR = /usr/local/ioapi_3.1
(2) SRCDIR   = $work/ioapi

$ make >& ioapi.log (s)
$ find / -name "libioapi*" (s)
##===============================IOAPI Build End========================================================
## run as superuser only at (s) mark; other than that use regular user
##===============================CMAQ Build Start=======================================================
##======================================================================================================
$ tar -xvf CMAQv5.tar
$ cd CMAQv5.0/scripts
$ cp $work/config.cmaq ./
$ mkdir -p $M3LIB
$ cd $M3LIB
## Link netcdf, mpich, ioapi libs for my case i use the following
$ ln -s /usr/local netcdf
$ mkdir ioapi_3.1
$ cd ioapi_3.1
$ ln -s /usr/local/ioapi_3.1/Linux2_x86_64pg Linux2_x86_64pgf
$ ln -s /usr/local mpich

## Build bldmake which is needed to extract other scripts
$ cd $M3HOME/scripts/build
$ ./bldit.bldmake >& bld.log

## Build STENEX for parallel and serial processing
$ source $M3HOME/scripts/config.cmaq
$ cd $M3HOME/scripts/stenex
$ ./bldit.se >& bld_se
$ ./bldit.se_noop >& bld_noop

## Build Pario for parallel CCTM operation
$ cd $M3HOME/scripts/pario
$ source $M3HOME/scripts/config.cmaq
## Change in the script >> set IOAPIEXT = $work/ioapi/fixed_src
$ ./bldit.pario >& bld.log

## Build icon for initial condition of the model
$ source $M3HOME/scripts/config.cmaq
$ cd $M3HOME/scripts/icon
## Change in bldit.icon >>
(1) set NETCDF = "${M3LIB}/netcdf/lib -lnetcdff -lnetcdf"
$ ./bldit.icon >& bld.log
## ICON_V5f_Linux2_x86_64pgf executable will be created

## Build bcon for boundary condition of the model
$ source $M3HOME/scripts/config.cmaq
$ cd $M3HOME/scripts/bcon
## Change in bldit.bcon >>
(1) set NETCDF = "${M3LIB}/netcdf/lib -lnetcdff -lnetcdf"
$ ./bldit.bcon >& bld.log
## BCON_V5f_Linux2_x86_64pgf executable will be created

## Build CCTM (CMAQ Chemistry Transport Model)
$ source $M3HOME/scripts/config.cmaq
$ cd $M3HOME/scripts/cctm
$ yum install librdmacm-static.x86_64 (s)
$ yum install libibumad-static.x86_64 (s)
## also get these two libs >> opa & mpl
## Change in bldit.cctm >>
(1) set NETCDF = "${M3LIB}/netcdf/lib -lnetcdff -lnetcdf"
$ ./bldit.cctm >& bld.log
## CCTM_V5f_Linux2_x86_64pgf execution file should be created

## Build jproc to produce photolysis rates for the mechanism
$ source $M3HOME/scripts/config.cmaq
$ cd $M3HOME/scripts/jproc
## Change in bldit.jproc >>
(1) set NETCDF = "${M3LIB}/netcdf/lib -lnetcdff -lnetcdf"
$ ./bldit.jproc >& bld.log
## JPROC_V5f_Linux2_x86_64pgf execution file should be created

## MCIP is required to pre-process the meteorological data to feed into CCTM
$ cd $M3HOME/scripts/mcip4/src
$ source $M3HOME/scripts/config.cmaq
$ yum install pvm.x86_64 (s)
 ## Change in Makefile >>
(1) NETCDF =  ${M3LIB}/netcdf
(2) IOAPI_ROOT = ${M3LIB}/ioapi_3.1
(3) PVM = /usr/share/pvm3/src/LINUXX86_64
(4) uncomment portland group fortran
(5) uncomment 2nd FFLAGS
(6) in FFLAGS -I$(IOAPI_ROOT)/Linux2_x86_64pgf
(7) in LIBS -L$(PVM) -lpvm3
(8) FC = pgf90
$ make
## Executable mcip.exe should be created
##======================================CMAQ Build End===========================================
##=================================================================================================
##======================================CMAQ Benchmarking Start=========================================
## run icon for initial conditions
$ source $M3HOME/scripts/config.cmaq
$ tar -xvf $work/DATA.CMAQv5.tar
$ cd $M3HOME/scripts/icon
## in run.icon comment 4 lines below species defn
$ ln -s $M3HOME/scripts/icon/BLD_V5f/GC_cb05tucl_ae6_aq.nml gc_matrix.nml
$ ln -s $M3HOME/scripts/icon/BLD_V5f/AE_cb05tucl_ae6_aq.nml ae_matrix.nml
$ ln -s $M3HOME/scripts/icon/BLD_V5f/NR_cb05tucl_ae6_aq.nml nr_matrix.nml
$ ln -s $M3HOME/scripts/icon/BLD_V5f/Species_Table_TR_0.nml tr_matrix.nml
$ export LD_LIBRARY_PATH=/usr/local/lib
$ ./run.icon >& run.log

## run bcon for boundary conditions
$ source $M3HOME/scripts/config.cmaq
$ cd $M3HOME/scripts/bcon
## in run.bcon comment 4 lines below species def
$ ln -s $M3HOME/scripts/bcon/BLD_V5f/GC_cb05tucl_ae6_aq.nml gc_matrix.nml
$ ln -s $M3HOME/scripts/bcon/BLD_V5f/AE_cb05tucl_ae6_aq.nml ae_matrix.nml
$ ln -s $M3HOME/scripts/bcon/BLD_V5f/NR_cb05tucl_ae6_aq.nml nr_matrix.nml
$ ln -s $M3HOME/scripts/bcon/BLD_V5f/Species_Table_TR_0.nml tr_matrix.nml
$ ./run.bcon >& run.log

## run CCTM for parallel processing
$ cd $M3HOME/scripts/cctm
$ source $M3HOME/scripts/config.cmaq
## in run.cctm change the following >>
(1) setenv NPCOL_NPROW "4 2"; set NPROCS   = 8
(2) # time  $BASE/$EXEC
(3)  set MPI = /usr/local/bin
     set MPIRUN = $MPI/mpiexec
     time $MPIRUN -n $NPROCS $BASE/$EXEC
(4) comment 4 lines below species def
$ ln -s $M3HOME/scripts/cctm/BLD_V5f/GC_cb05tucl_ae6_aq.nml gc_matrix.nml
$ ln -s $M3HOME/scripts/cctm/BLD_V5f/AE_cb05tucl_ae6_aq.nml ae_matrix.nml
$ ln -s $M3HOME/scripts/cctm/BLD_V5f/NR_cb05tucl_ae6_aq.nml nr_matrix.nml
$ ln -s $M3HOME/scripts/cctm/BLD_V5f/Species_Table_TR_0.nml tr_matrix.nml
$ ./run.cctm >& run.log
## This should start running in 8 processors (can be seen with $top)
##=========================================CMAQ Benchmarking End====================================
##==========================================Verify Output===========================================
$ tar -xvf $work/DATA_REF.CMAQv5.tar
## this should produce the reference output in $M3DATA/ref
## we can verify for any data variable with ncdump -v variable output.file
$ ncdump -v CO ICON_V5f_CMAQ-BENCHMARK_profile |less