Merge pull request #224 from ucb-bar/dev-sh

Improve robustness/portability of toolchain shell scripts

scripts/build-static-libfesvr.sh

@@ -11,7 +11,11 @@ fi
 
 set -e
 
-objs=$(head -n 1 <(make -f <( echo -e 'include Makefile\n$(info $(value fesvr_objs))') -n))
+objs=$(make -n -f <(
-ar rcs -o libfesvr.a $objs
+    echo 'include Makefile'
-cp -f libfesvr.a $RISCV/lib
+    echo '$(info $(value fesvr_objs))'
+    ) | head -n 1)
+
+ar rcs -o libfesvr.a $objs
+cp -f libfesvr.a "${RISCV}/lib"
 

scripts/build-toolchains.sh

@@ -12,82 +12,82 @@ RDIR=$(pwd)
 
 PRECOMPILED_REPO_HASH=56a40961c98db5e8f904f15dc6efd0870bfefd9e
 
-function usage
+usage() {
-{
+    echo "usage: ${0} [riscv-tools | esp-tools | ec2fast]"
-    echo "usage: ./scripts/build-toolchains.sh [riscv-tools] [esp-tools] [ec2fast | --ec2fast] "
     echo "   riscv: if set, builds the riscv toolchain (this is also the default)"
     echo "   hwacha: if set, builds esp-tools toolchain"
     echo "   ec2fast: if set, pulls in a pre-compiled RISC-V toolchain for an EC2 manager instance"
+    exit "$1"
+}
+
+error() {
+    echo "${0##*/}: ${1}" >&2
 }
 
 #taken from riscv-tools to check for open-ocd autoconf versions
 check_version() {
-    $1 --version | awk "NR==1 {if (\$NF>$2) {exit 0} exit 1}" || (
+    "$1" --version | awk "NR==1 {if (\$NF>$2) {exit 0} exit 1}" || {
-        echo $3 requires at least version $2 of $1. Aborting.
+        error "${3} requires at least ${1} version ${2}"
         exit 1
-    )
+    }
 }
 
-if [ "$1" == "--help" -o "$1" == "-h" -o "$1" == "-H" ]; then
-    usage
-    exit 3
-fi
-
 TOOLCHAIN="riscv-tools"
 EC2FASTINSTALL="false"
 FASTINSTALL="false"
-while test $# -gt 0
+
-do
+while getopts 'hH-:' opt ; do
-   case "$1" in
+    case $opt in
-        riscv-tools)
+    h|H)
-            TOOLCHAIN="riscv-tools"
+        usage 3 ;;
-            ;;
+    -)
-        esp-tools)
+        case $OPTARG in
-            TOOLCHAIN="esp-tools"
+        help)
-            ;;
+            usage 3 ;;
-        ec2fast | --ec2fast) # I don't want to break this api
+        ec2fast) # Preserve compatibility
-            EC2FASTINSTALL=true
+            EC2FASTINSTALL=true ;;
-            ;;
+        *)
-        -h | -H | --help)
+            error "invalid option: --${OPTARG}"
-            usage
+            usage 1 ;;
-            exit 3
+        esac ;;
-            ;;
+    *)
-        --*) echo "ERROR: bad option $1"
+        error "invalid option: -${opt}"
-            usage
+        usage 1 ;;
-            exit 1
-            ;;
-        *) echo "ERROR: bad argument $1"
-            usage
-            exit 2
-            ;;
     esac
-    shift
 done
 
+shift $((OPTIND - 1))
+
+if [ "$1" = ec2fast ] ; then
+    EC2FASTINSTALL=true
+elif [ -n "$1" ] ; then
+    TOOLCHAIN="$1"
+fi
+
 
 if [ "$EC2FASTINSTALL" = "true" ]; then
     if [ "$TOOLCHAIN" = "riscv-tools" ]; then
-      cd $RDIR
+      cd "$RDIR"
       git clone https://github.com/firesim/firesim-riscv-tools-prebuilt.git
       cd firesim-riscv-tools-prebuilt
-      git checkout $PRECOMPILED_REPO_HASH
+      git checkout "$PRECOMPILED_REPO_HASH"
       PREBUILTHASH="$(cat HASH)"
-      git -C $CHIPYARD_DIR submodule update --init  toolchains/$TOOLCHAIN
+      git -C "${CHIPYARD_DIR}" submodule update --init "toolchains/${TOOLCHAIN}"
       cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN"
       GITHASH="$(git rev-parse HEAD)"
-      cd $RDIR
+      cd "$RDIR"
       echo "prebuilt hash: $PREBUILTHASH"
       echo "git      hash: $GITHASH"
       if [[ $PREBUILTHASH == $GITHASH && "$EC2FASTINSTALL" == "true" ]]; then
           FASTINSTALL=true
           echo "Using fast pre-compiled install for riscv-tools"
       else
-          echo "Error: hash of precompiled toolchain doesn't match the riscv-tools submodule hash."
+          error 'error: hash of precompiled toolchain does not match the riscv-tools submodule hash'
-          exit
+          exit -1
       fi
     else
-          echo "Error: No precompiled toolchain for esp-tools or other non-native riscv-tools."
+          error "error: unsupported precompiled toolchain: ${TOOLCHAIN}"
-          exit 
+          exit -1
     fi
 fi
 
@@ -104,44 +104,56 @@ if [ "$FASTINSTALL" = true ]; then
     mv distrib "$RISCV"
     # copy HASH in case user wants it later
     cp HASH "$RISCV"
-    cd $RDIR
+    cd "$RDIR"
     rm -rf firesim-riscv-tools-prebuilt
 else
     mkdir -p "$RISCV"
-    git -C $CHIPYARD_DIR submodule update --init --recursive toolchains/$TOOLCHAIN #--jobs 8
+    git -C "${CHIPYARD_DIR}" submodule update --init --recursive "toolchains/${TOOLCHAIN}" #--jobs 8
     cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN"
-    export MAKEFLAGS="-j16"
+
+    # Scale number of parallel make jobs by hardware thread count
+    ncpu="$(getconf _NPROCESSORS_ONLN || # GNU
+        getconf NPROCESSORS_ONLN || # *BSD, Solaris
+        nproc --all || # Linux
+        sysctl -n hw.ncpu || # *BSD, OS X
+        :)" 2>/dev/null
+    case ${ncpu} in
+    ''|*[^0-9]*) ;; # Ignore non-integer values
+    *) export MAKEFLAGS="-j ${ncpu}" ;;
+    esac
+
     #build the actual toolchain
     #./build.sh
     source build.common
     echo "Starting RISC-V Toolchain build process"
-    build_project riscv-fesvr --prefix=$RISCV
+    build_project riscv-fesvr --prefix="${RISCV}"
-    build_project riscv-isa-sim --prefix=$RISCV --with-fesvr=$RISCV
+    build_project riscv-isa-sim --prefix="${RISCV}" --with-fesvr="${RISCV}"
-    build_project riscv-gnu-toolchain --prefix=$RISCV
+    build_project riscv-gnu-toolchain --prefix="${RISCV}"
-    CC= CXX= build_project riscv-pk --prefix=$RISCV --host=riscv64-unknown-elf
+    CC= CXX= build_project riscv-pk --prefix="${RISCV}" --host=riscv64-unknown-elf
-    build_project riscv-tests --prefix=$RISCV/riscv64-unknown-elf
+    build_project riscv-tests --prefix="${RISCV}/riscv64-unknown-elf"
     echo -e "\\nRISC-V Toolchain installation completed!"
 
     # build static libfesvr library for linking into firesim driver (or others)
     cd riscv-fesvr/build
-    $CHIPYARD_DIR/scripts/build-static-libfesvr.sh
+    "${CHIPYARD_DIR}/scripts/build-static-libfesvr.sh"
-    cd $RDIR
+    cd "$RDIR"
     # build linux toolchain
     cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN/riscv-gnu-toolchain/build"
-    make -j16 linux
+    make linux
     echo -e "\\nRISC-V Linux GNU Toolchain installation completed!"
 
 fi
 
-cd $RDIR
+cd "$RDIR"
 
-echo "export CHIPYARD_TOOLCHAIN_SOURCED=1" > env.sh
+{
-echo "export RISCV=$RISCV" >> env.sh
+    echo "export CHIPYARD_TOOLCHAIN_SOURCED=1"
-echo "export PATH=$RISCV/bin:$RDIR/$DTCversion:\$PATH" >> env.sh
+    echo "export RISCV=$(printf '%q' "$RISCV")"
-echo "export LD_LIBRARY_PATH=$RISCV/lib\${LD_LIBRARY_PATH:+":${LD_LIBRARY_PATH}"}" >> env.sh
+    echo "export PATH=\${RISCV}/bin:\${PATH}"
+    echo "export LD_LIBRARY_PATH=\${RISCV}/lib\${LD_LIBRARY_PATH:+":\${LD_LIBRARY_PATH}"}"
+} > env.sh
 echo "Toolchain Build Complete!"
 
-
 if [ "$FASTINSTALL" = "false" ]; then
     # commands that can't run on EC2 (specifically, OpenOCD because of autoconf version_
     # see if the instance info page exists. if not, we are not on ec2.
@@ -153,8 +165,8 @@ if [ "$FASTINSTALL" = "false" ]; then
         cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN"
         check_version automake 1.14 "OpenOCD build"
         check_version autoconf 2.64 "OpenOCD build"
-        build_project riscv-openocd --prefix=$RISCV --enable-remote-bitbang --enable-jtag_vpi --disable-werror
+        build_project riscv-openocd --prefix="${RISCV}" --enable-remote-bitbang --enable-jtag_vpi --disable-werror
         echo -e "\\nRISC-V OpenOCD installation completed!"
-        cd $RDIR
+        cd "$RDIR"
     fi
 fi

scripts/check-tracegen.sh

@@ -13,11 +13,11 @@ AXE_SHRINK=${AXE_DIR}/src/axe-shrink.py
 PATH=$PATH:${AXE_DIR}/src
 
 grep '.*:.*#.*@' $1 > /tmp/clean-trace.txt
-$TO_AXE /tmp/clean-trace.txt > /tmp/trace.axe
+"$TO_AXE" /tmp/clean-trace.txt > /tmp/trace.axe
-result=$($AXE check wmo /tmp/trace.axe)
+result=$("$AXE" check wmo /tmp/trace.axe)
 
-if [ $result != "OK" ]; then
+if [ "$result" != OK ]; then
-    $AXE_SHRINK wmo /tmp/trace.axe
+    "$AXE_SHRINK" wmo /tmp/trace.axe
 else
-    echo "OK"
+    echo OK
 fi

scripts/firesim-setup.sh

@@ -8,11 +8,11 @@ set -o pipefail
 RDIR=$(pwd)
 scripts_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-cd $scripts_dir/..
+cd "${scripts_dir}/.."
 
 # Reenable the FireSim submodule
 git config --unset submodule.sims/firesim.update || true
 git submodule update --init sims/firesim
 cd sims/firesim
-./build-setup.sh $@ --library
+./build-setup.sh "$@" --library
-cd $RDIR
+cd "$RDIR"

scripts/init-submodules-no-riscv-tools.sh

@@ -28,9 +28,9 @@ git config --unset submodule.vlsi/hammer-cad-plugins.update
 # Renable firesim and init only the required submodules to provide
 # all required scala deps, without doing a full build-setup
 git config --unset submodule.sims/firesim.update
-cd $scripts_dir/../sims/
+cd "${scripts_dir}/../sims"
 git submodule update --init firesim
 cd firesim/sim
 git submodule update --init midas
-cd $RDIR
+cd "$RDIR"
 git config submodule.sims/firesim.update none

scripts/init-vlsi.sh

@@ -3,11 +3,9 @@
 set -e
 set -o pipefail
 
-
-
 # Initialize HAMMER and CAD-plugins
 git submodule update --init --recursive vlsi/hammer
 git submodule update --init --recursive vlsi/hammer-cad-plugins
 
 # Initialize HAMMER tech plugin
-git submodule update --init --recursive vlsi/hammer-$1-plugin
+git submodule update --init --recursive vlsi/hammer-"$1"-plugin