From 9e7c5b3312f9528539b73954b318f7f9832641a9 Mon Sep 17 00:00:00 2001
From: Wirawan Purwanto <wpurwant@turing.hpc.odu.edu>
Date: Tue, 20 Apr 2021 00:28:39 -0400
Subject: [PATCH] * Modularize & tidied up the nwbuild05 script.

---
 .../nwbuild05-OPENIB-internal-blas-lapack.sh  | 112 ++++++++++++++----
 1 file changed, 87 insertions(+), 25 deletions(-)

diff --git a/wahab/builds/nwchem/nwbuild05-OPENIB-internal-blas-lapack.sh b/wahab/builds/nwchem/nwbuild05-OPENIB-internal-blas-lapack.sh
index e40c2f4..ff17371 100755
--- a/wahab/builds/nwchem/nwbuild05-OPENIB-internal-blas-lapack.sh
+++ b/wahab/builds/nwchem/nwbuild05-OPENIB-internal-blas-lapack.sh
@@ -1,18 +1,78 @@
 #!/bin/bash
 #
+# build05:
+# * Use OPENIB ARMCI backend
+# * Less fancy build: do not use external LAPACK or BLAS
+#
 # 20210414 - First version, forked off nwbuild04-ARMCI-MPI-internal-blas-lapack.sh
 
+preamble () {
+    # Site-specific preambles
+
     module load ucx/1.9 openmpi/3.1.4
 
+    SRC_ROOT_DIR=/cm/shared/containers/srcs/nwchem
+    GA_PACKAGE=ga-5.7.1
+    SRC_TARBALL_NWCHEM=$SRC_ROOT_DIR/v7.0.0-release.tar.gz
+    SRC_TARBALL_GA=$SRC_ROOT_DIR/$GA_PACKAGE.tar.gz
+
     #export NWCHEM_TOP=/opt/nwchem.build04c
     # FOR TESTINGS --> shortcut to the above path due to path cannot exceed 64 chars :(
     export NWCHEM_TOP=/scratch/wpurwant/BUILD/nwchem7/gcc7.3-openmpi3.1.4/nwchem.OPENIB  # FOR TESTINGS
-    export NWCHEM_TOP=/scratch/wpurwant/.L/nwchem.OPENIB
+    #export NWCHEM_TOP=/scratch/wpurwant/.L/nwchem.OPENIB   # temp shortlink (workaround)--not needed anymore
+
+    # FOR PRODUCTION
+    export NWCHEM_TOP=/scratch/wpurwant/BUILD/nwchem7/gcc7.3-openmpi3.1.4/prod-nwchem.OPENIB
+
+    if false; then # build production target--not used
+        export NWCHEM_TOP=/shared/apps/manual/nwchem/7.0.0/gcc7.3-openmpi3.1.4/BUILD-OPENIB
+        # This tweak was needed to build faster (files are actually located on scratch)
+        BUILD_TOP=/scratch/wpurwant/BUILD/nwchem7/gcc7.3-openmpi3.1.4/prod-nwchem.OPENIB
+        mkdir -p $BUILD_TOP
+        if [ ! -e $NWCHEM_TOP ]; then
+            ln -s $BUILD_TOP $NWCHEM_TOP
+        fi
+    fi
 
+    # Build tweaks -- these are VERY IMPORTANT to set correctly
     export NWCHEM_MODULES="all"  # skip "python"
     export NWCHEM_TARGET=LINUX64
 
-    if false; then
+    export USE_MPI=y
+    export USE_NOIO=y
+    export USE_SCALAPACK=n
+    export USE_PYTHON64=n
+
+    export ARMCI_NETWORK=OPENIB
+
+    # Set to "y" to FORCE internal BLAS & LAPACK:
+    export USE_INTERNALBLAS=y
+    export SCALAPACK_SIZE=8
+    export BLAS_SIZE=8
+
+    #export SCALAPACK="-lmkl_scalapack_ilp64 -lmkl_gf_ilp64 -lmkl_gnu_thread -lmkl_core -lmkl_blacs_intelmpi_ilp64 -lgomp -lpthread -lm -ldl"
+    #export SCALAPACK_LIB="$SCALAPACK"
+
+    #export BLASOPT="-lmkl_gf_ilp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl"
+    #export LAPACK_LIB=$BLASOPT
+
+    # set OPT_COMPILER_NAME_WRAPPERS to yes if needing "proper" program names
+    # for the compiler front-ends:
+    OPT_COMPILER_NAME_WRAPPERS=no
+
+    # automatically determined, no need to change
+    OPT_USE_PYTHON=$(if echo " $NWCHEM_MODULES " | grep -q " python "; then echo yes; else echo no; fi)
+
+    # Needed to accommodate long path of nwchem source: set to capital "Y" only
+    # see src/utils/GNUmakefile around line 19
+    export NWCHEM_LONG_PATHS=Y
+}
+
+build_nwchem_native () {
+    # Function to build nwchem in native (non-container) settings.
+    # This script builds with GCC (hardcoded)
+
+    if [ "x$OPT_COMPILER_NAME_WRAPPERS" = xyes ]; then
         COMPILER_DIR=$NWCHEM_TOP/compilers
         mkdir -p $COMPILER_DIR
         if ! test -e $COMPILER_DIR/gcc; then ln -sf $(which $CC) $COMPILER_DIR/gcc; fi
@@ -20,7 +80,7 @@
         if ! test -e $COMPILER_DIR/gfortran; then ln -sf $(which $FC) $COMPILER_DIR/gfortran; fi
 
         # Make this THE path to find gcc, g++, and gfortran:
-        PATH=$COMPILER_DIR:$PATH
+        export PATH="$COMPILER_DIR:$PATH"
     fi
 
     # Override the compiler's names to point to the CONDA-provided toolchains
@@ -29,30 +89,15 @@
     export FC=gfortran
     echo PATH=$PATH
 
+    (
+    set -x
     mkdir -p $NWCHEM_TOP
-    test -d $NWCHEM_TOP/src || tar -C $NWCHEM_TOP -xf /cm/shared/containers/srcs/nwchem/v7.0.0-release.tar.gz --strip-components=1
-    test -d $NWCHEM_TOP/src/tools/ga-5.7.1 || tar -C $NWCHEM_TOP/src/tools -xf /cm/shared/containers/srcs/nwchem/ga-5.7.1.tar.gz
-
-    # scale down: do not use external LAPACK or BLAS
-    export USE_MPI=y
-    export USE_NOIO=y
-    export USE_SCALAPACK=n
-    export USE_PYTHON64=n
-    # FORCE:
-    export USE_INTERNALBLAS=y
-
-    export ARMCI_NETWORK=OPENIB
+    test -d $NWCHEM_TOP/src || tar -C $NWCHEM_TOP -xf $SRC_TARBALL_NWCHEM --strip-components=1
+    test -d $NWCHEM_TOP/src/tools/$GA_PACKAGE || tar -C $NWCHEM_TOP/src/tools -xf $SRC_TARBALL_GA
+    )
 
-    export SCALAPACK_SIZE=8
-    export BLAS_SIZE=8
-
-    #export SCALAPACK="-lmkl_scalapack_ilp64 -lmkl_gf_ilp64 -lmkl_gnu_thread -lmkl_core -lmkl_blacs_intelmpi_ilp64 -lgomp -lpthread -lm -ldl"
-    #export SCALAPACK_LIB="$SCALAPACK"
-
-    #export BLASOPT="-lmkl_gf_ilp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl"
-    #export LAPACK_LIB=$BLASOPT
-
-    if false; then
+    if [ "x$OPT_USE_PYTHON" = xyes ]; then
+        # FIXME. Not tried yet.
         export PYTHONHOME=/opt/conda
         export PYTHONVERSION=2.7
         (
@@ -80,3 +125,20 @@
     echo "PATH = $PATH"
 
     make
+}
+
+
+preamble
+
+if [ x"$1" = xpreamble ]; then
+    return
+    export
+    exit 1
+fi
+
+if [ x"$1" = x  -o  x"$1" = xbuild ]; then
+    build_nwchem_native
+else
+    echo "Invalid subcommand: $1" >&2
+    exit 2
+fi