From 40a94dd08e2ae2f8d54b06430907723e0e43a4bf Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Sat, 31 Aug 2019 01:14:39 -0700 Subject: [PATCH 1/6] scripts: Parse options with getopts builtin --- scripts/build-toolchains.sh | 66 ++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/scripts/build-toolchains.sh b/scripts/build-toolchains.sh index e743afe4..03ddc8fa 100755 --- a/scripts/build-toolchains.sh +++ b/scripts/build-toolchains.sh @@ -12,12 +12,16 @@ RDIR=$(pwd) PRECOMPILED_REPO_HASH=56a40961c98db5e8f904f15dc6efd0870bfefd9e -function usage -{ - echo "usage: ./scripts/build-toolchains.sh [riscv-tools] [esp-tools] [ec2fast | --ec2fast] " +usage() { + echo "usage: ${0} [riscv-tools | esp-tools | ec2fast]" echo " riscv: if set, builds the riscv toolchain (this is also the default)" echo " hwacha: if set, builds esp-tools toolchain" echo " ec2fast: if set, pulls in a pre-compiled RISC-V toolchain for an EC2 manager instance" + exit "$1" +} + +error() { + echo "${0##*/}: ${1}" >&2 } #taken from riscv-tools to check for open-ocd autoconf versions @@ -28,42 +32,38 @@ check_version() { ) } -if [ "$1" == "--help" -o "$1" == "-h" -o "$1" == "-H" ]; then - usage - exit 3 -fi - TOOLCHAIN="riscv-tools" EC2FASTINSTALL="false" FASTINSTALL="false" -while test $# -gt 0 -do - case "$1" in - riscv-tools) - TOOLCHAIN="riscv-tools" - ;; - esp-tools) - TOOLCHAIN="esp-tools" - ;; - ec2fast | --ec2fast) # I don't want to break this api - EC2FASTINSTALL=true - ;; - -h | -H | --help) - usage - exit 3 - ;; - --*) echo "ERROR: bad option $1" - usage - exit 1 - ;; - *) echo "ERROR: bad argument $1" - usage - exit 2 - ;; + +while getopts 'hH-:' opt ; do + case $opt in + h|H) + usage 3 ;; + -) + case $OPTARG in + help) + usage 3 ;; + ec2fast) # Preserve compatibility + EC2FASTINSTALL=true ;; + *) + error "invalid option: --${OPTARG}" + usage 1 ;; + esac ;; + *) + error "invalid option: -${opt}" + usage 1 ;; esac - shift done +shift $((OPTIND - 1)) + +if [ "$1" = ec2fast ] ; then + EC2FASTINSTALL=true +elif [ -z "$1" ] ; then + TOOLCHAIN="$1" +fi + if [ "$EC2FASTINSTALL" = "true" ]; then if [ "$TOOLCHAIN" = "riscv-tools" ]; then From 10b708d1732d5c2832dd06991118bec91738ff59 Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Sat, 31 Aug 2019 01:25:56 -0700 Subject: [PATCH 2/6] scripts: Ensure non-zero exit code on error --- scripts/build-toolchains.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build-toolchains.sh b/scripts/build-toolchains.sh index 03ddc8fa..c1d1da62 100755 --- a/scripts/build-toolchains.sh +++ b/scripts/build-toolchains.sh @@ -82,12 +82,12 @@ if [ "$EC2FASTINSTALL" = "true" ]; then FASTINSTALL=true echo "Using fast pre-compiled install for riscv-tools" else - echo "Error: hash of precompiled toolchain doesn't match the riscv-tools submodule hash." - exit + error 'error: hash of precompiled toolchain does not match the riscv-tools submodule hash' + exit -1 fi else - echo "Error: No precompiled toolchain for esp-tools or other non-native riscv-tools." - exit + error "error: unsupported precompiled toolchain: ${TOOLCHAIN}" + exit -1 fi fi From 2c2f5a76e3f1648b77c56a68b803d8b096770697 Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Sat, 31 Aug 2019 01:52:51 -0700 Subject: [PATCH 3/6] scripts: Fix quoting to prevent unintended word splitting --- scripts/build-static-libfesvr.sh | 2 +- scripts/build-toolchains.sh | 40 +++++++++++------------ scripts/check-tracegen.sh | 10 +++--- scripts/firesim-setup.sh | 6 ++-- scripts/init-submodules-no-riscv-tools.sh | 4 +-- scripts/init-vlsi.sh | 4 +-- 6 files changed, 32 insertions(+), 34 deletions(-) diff --git a/scripts/build-static-libfesvr.sh b/scripts/build-static-libfesvr.sh index 360028ad..96724bd6 100755 --- a/scripts/build-static-libfesvr.sh +++ b/scripts/build-static-libfesvr.sh @@ -13,5 +13,5 @@ set -e objs=$(head -n 1 <(make -f <( echo -e 'include Makefile\n$(info $(value fesvr_objs))') -n)) ar rcs -o libfesvr.a $objs -cp -f libfesvr.a $RISCV/lib +cp -f libfesvr.a "${RISCV}/lib" diff --git a/scripts/build-toolchains.sh b/scripts/build-toolchains.sh index c1d1da62..8e0d40e7 100755 --- a/scripts/build-toolchains.sh +++ b/scripts/build-toolchains.sh @@ -26,10 +26,10 @@ error() { #taken from riscv-tools to check for open-ocd autoconf versions check_version() { - $1 --version | awk "NR==1 {if (\$NF>$2) {exit 0} exit 1}" || ( - echo $3 requires at least version $2 of $1. Aborting. + "$1" --version | awk "NR==1 {if (\$NF>$2) {exit 0} exit 1}" || { + error "${3} requires at least ${1} version ${2}" exit 1 - ) + } } TOOLCHAIN="riscv-tools" @@ -60,22 +60,22 @@ shift $((OPTIND - 1)) if [ "$1" = ec2fast ] ; then EC2FASTINSTALL=true -elif [ -z "$1" ] ; then +elif [ -n "$1" ] ; then TOOLCHAIN="$1" fi if [ "$EC2FASTINSTALL" = "true" ]; then if [ "$TOOLCHAIN" = "riscv-tools" ]; then - cd $RDIR + cd "$RDIR" git clone https://github.com/firesim/firesim-riscv-tools-prebuilt.git cd firesim-riscv-tools-prebuilt - git checkout $PRECOMPILED_REPO_HASH + git checkout "$PRECOMPILED_REPO_HASH" PREBUILTHASH="$(cat HASH)" - git -C $CHIPYARD_DIR submodule update --init toolchains/$TOOLCHAIN + git -C "${CHIPYARD_DIR}" submodule update --init "toolchains/${TOOLCHAIN}" cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN" GITHASH="$(git rev-parse HEAD)" - cd $RDIR + cd "$RDIR" echo "prebuilt hash: $PREBUILTHASH" echo "git hash: $GITHASH" if [[ $PREBUILTHASH == $GITHASH && "$EC2FASTINSTALL" == "true" ]]; then @@ -104,28 +104,28 @@ if [ "$FASTINSTALL" = true ]; then mv distrib "$RISCV" # copy HASH in case user wants it later cp HASH "$RISCV" - cd $RDIR + cd "$RDIR" rm -rf firesim-riscv-tools-prebuilt else mkdir -p "$RISCV" - git -C $CHIPYARD_DIR submodule update --init --recursive toolchains/$TOOLCHAIN #--jobs 8 + git -C "${CHIPYARD_DIR}" submodule update --init --recursive "toolchains/${TOOLCHAIN}" #--jobs 8 cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN" export MAKEFLAGS="-j16" #build the actual toolchain #./build.sh source build.common echo "Starting RISC-V Toolchain build process" - build_project riscv-fesvr --prefix=$RISCV - build_project riscv-isa-sim --prefix=$RISCV --with-fesvr=$RISCV - build_project riscv-gnu-toolchain --prefix=$RISCV - CC= CXX= build_project riscv-pk --prefix=$RISCV --host=riscv64-unknown-elf - build_project riscv-tests --prefix=$RISCV/riscv64-unknown-elf + build_project riscv-fesvr --prefix="${RISCV}" + build_project riscv-isa-sim --prefix="${RISCV}" --with-fesvr="${RISCV}" + build_project riscv-gnu-toolchain --prefix="${RISCV}" + CC= CXX= build_project riscv-pk --prefix="${RISCV}" --host=riscv64-unknown-elf + build_project riscv-tests --prefix="${RISCV}/riscv64-unknown-elf" echo -e "\\nRISC-V Toolchain installation completed!" # build static libfesvr library for linking into firesim driver (or others) cd riscv-fesvr/build - $CHIPYARD_DIR/scripts/build-static-libfesvr.sh - cd $RDIR + "${CHIPYARD_DIR}/scripts/build-static-libfesvr.sh" + cd "$RDIR" # build linux toolchain cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN/riscv-gnu-toolchain/build" make -j16 linux @@ -133,7 +133,7 @@ else fi -cd $RDIR +cd "$RDIR" echo "export CHIPYARD_TOOLCHAIN_SOURCED=1" > env.sh echo "export RISCV=$RISCV" >> env.sh @@ -153,8 +153,8 @@ if [ "$FASTINSTALL" = "false" ]; then cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN" check_version automake 1.14 "OpenOCD build" check_version autoconf 2.64 "OpenOCD build" - build_project riscv-openocd --prefix=$RISCV --enable-remote-bitbang --enable-jtag_vpi --disable-werror + build_project riscv-openocd --prefix="${RISCV}" --enable-remote-bitbang --enable-jtag_vpi --disable-werror echo -e "\\nRISC-V OpenOCD installation completed!" - cd $RDIR + cd "$RDIR" fi fi diff --git a/scripts/check-tracegen.sh b/scripts/check-tracegen.sh index 6462613c..bc82f572 100755 --- a/scripts/check-tracegen.sh +++ b/scripts/check-tracegen.sh @@ -13,11 +13,11 @@ AXE_SHRINK=${AXE_DIR}/src/axe-shrink.py PATH=$PATH:${AXE_DIR}/src grep '.*:.*#.*@' $1 > /tmp/clean-trace.txt -$TO_AXE /tmp/clean-trace.txt > /tmp/trace.axe -result=$($AXE check wmo /tmp/trace.axe) +"$TO_AXE" /tmp/clean-trace.txt > /tmp/trace.axe +result=$("$AXE" check wmo /tmp/trace.axe) -if [ $result != "OK" ]; then - $AXE_SHRINK wmo /tmp/trace.axe +if [ "$result" != OK ]; then + "$AXE_SHRINK" wmo /tmp/trace.axe else - echo "OK" + echo OK fi diff --git a/scripts/firesim-setup.sh b/scripts/firesim-setup.sh index 302294bf..0787fd3c 100755 --- a/scripts/firesim-setup.sh +++ b/scripts/firesim-setup.sh @@ -8,11 +8,11 @@ set -o pipefail RDIR=$(pwd) scripts_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd $scripts_dir/.. +cd "${scripts_dir}/.." # Reenable the FireSim submodule git config --unset submodule.sims/firesim.update || true git submodule update --init sims/firesim cd sims/firesim -./build-setup.sh $@ --library -cd $RDIR +./build-setup.sh "$@" --library +cd "$RDIR" diff --git a/scripts/init-submodules-no-riscv-tools.sh b/scripts/init-submodules-no-riscv-tools.sh index 3a95f5c6..8cd6ced3 100755 --- a/scripts/init-submodules-no-riscv-tools.sh +++ b/scripts/init-submodules-no-riscv-tools.sh @@ -28,9 +28,9 @@ git config --unset submodule.vlsi/hammer-cad-plugins.update # Renable firesim and init only the required submodules to provide # all required scala deps, without doing a full build-setup git config --unset submodule.sims/firesim.update -cd $scripts_dir/../sims/ +cd "${scripts_dir}/../sims" git submodule update --init firesim cd firesim/sim git submodule update --init midas -cd $RDIR +cd "$RDIR" git config submodule.sims/firesim.update none diff --git a/scripts/init-vlsi.sh b/scripts/init-vlsi.sh index 294f8628..89f9ac56 100755 --- a/scripts/init-vlsi.sh +++ b/scripts/init-vlsi.sh @@ -3,11 +3,9 @@ set -e set -o pipefail - - # Initialize HAMMER and CAD-plugins git submodule update --init --recursive vlsi/hammer git submodule update --init --recursive vlsi/hammer-cad-plugins # Initialize HAMMER tech plugin -git submodule update --init --recursive vlsi/hammer-$1-plugin \ No newline at end of file +git submodule update --init --recursive vlsi/hammer-"$1"-plugin From 9e9e7dd442ade2a0988a5ce03116928b0d79ce97 Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Sat, 31 Aug 2019 02:00:48 -0700 Subject: [PATCH 4/6] scripts: Fix latent issues with env.sh * Ensure shell metacharacters are quoted in RISCV variable assignment. * Avoid unnecessary expansion of $RISCV and $LD_LIBRARY_PATH at generation time. * Remove undefined usage of $DTCversion. * Simplify output redirection. --- scripts/build-toolchains.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/build-toolchains.sh b/scripts/build-toolchains.sh index 8e0d40e7..6d466ad7 100755 --- a/scripts/build-toolchains.sh +++ b/scripts/build-toolchains.sh @@ -135,13 +135,14 @@ fi cd "$RDIR" -echo "export CHIPYARD_TOOLCHAIN_SOURCED=1" > env.sh -echo "export RISCV=$RISCV" >> env.sh -echo "export PATH=$RISCV/bin:$RDIR/$DTCversion:\$PATH" >> env.sh -echo "export LD_LIBRARY_PATH=$RISCV/lib\${LD_LIBRARY_PATH:+":${LD_LIBRARY_PATH}"}" >> env.sh +{ + echo "export CHIPYARD_TOOLCHAIN_SOURCED=1" + echo "export RISCV=$(printf '%q' "$RISCV")" + echo "export PATH=\${RISCV}/bin:\${PATH}" + echo "export LD_LIBRARY_PATH=\${RISCV}/lib\${LD_LIBRARY_PATH:+":\${LD_LIBRARY_PATH}"}" +} > env.sh echo "Toolchain Build Complete!" - if [ "$FASTINSTALL" = "false" ]; then # commands that can't run on EC2 (specifically, OpenOCD because of autoconf version_ # see if the instance info page exists. if not, we are not on ec2. From cff1f6b11e398be0748c944c5f96268ee2f782ce Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Sat, 31 Aug 2019 03:11:46 -0700 Subject: [PATCH 5/6] scripts: Eliminate unnecessary process substitution --- scripts/build-static-libfesvr.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/build-static-libfesvr.sh b/scripts/build-static-libfesvr.sh index 96724bd6..d5b7f896 100755 --- a/scripts/build-static-libfesvr.sh +++ b/scripts/build-static-libfesvr.sh @@ -11,7 +11,11 @@ fi set -e -objs=$(head -n 1 <(make -f <( echo -e 'include Makefile\n$(info $(value fesvr_objs))') -n)) +objs=$(make -n -f <( + echo 'include Makefile' + echo '$(info $(value fesvr_objs))' + ) | head -n 1) + ar rcs -o libfesvr.a $objs cp -f libfesvr.a "${RISCV}/lib" From 4963792e98eafc3a8c086da85c2e0904b503c43c Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Sat, 31 Aug 2019 14:51:14 -0700 Subject: [PATCH 6/6] scripts: Scale number of make jobs by hardware thread count If the hardware thread/core count cannot be determined successfully, avoid setting the '-j' option for make. --- scripts/build-toolchains.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/build-toolchains.sh b/scripts/build-toolchains.sh index 6d466ad7..76680f79 100755 --- a/scripts/build-toolchains.sh +++ b/scripts/build-toolchains.sh @@ -110,7 +110,18 @@ else mkdir -p "$RISCV" git -C "${CHIPYARD_DIR}" submodule update --init --recursive "toolchains/${TOOLCHAIN}" #--jobs 8 cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN" - export MAKEFLAGS="-j16" + + # Scale number of parallel make jobs by hardware thread count + ncpu="$(getconf _NPROCESSORS_ONLN || # GNU + getconf NPROCESSORS_ONLN || # *BSD, Solaris + nproc --all || # Linux + sysctl -n hw.ncpu || # *BSD, OS X + :)" 2>/dev/null + case ${ncpu} in + ''|*[^0-9]*) ;; # Ignore non-integer values + *) export MAKEFLAGS="-j ${ncpu}" ;; + esac + #build the actual toolchain #./build.sh source build.common @@ -128,7 +139,7 @@ else cd "$RDIR" # build linux toolchain cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN/riscv-gnu-toolchain/build" - make -j16 linux + make linux echo -e "\\nRISC-V Linux GNU Toolchain installation completed!" fi