Merge pull request #1223 from ucb-bar/misc-sh-updates

Misc. updates to bash scripts

.github/actions/run-tests/action.yml

@@ -23,7 +23,7 @@ runs:
     - name: Init submodules (since only the RTL is cached)
       run: |
         conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-${{ inputs.toolchain }}
-        ./scripts/init-submodules-no-riscv-tools.sh --skip-validate
+        ./scripts/init-submodules-no-riscv-tools.sh --force
       shell: bash -leo pipefail {0}
 
     # Note: You shouldn't need the other inputs since it shouldn't build RTL from scratch

.github/scripts/remote-do-rtl-build.sh

@@ -15,7 +15,7 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 source $SCRIPT_DIR/defaults.sh
 
 cd $REMOTE_CHIPYARD_DIR
-./scripts/init-submodules-no-riscv-tools.sh --skip-validate
+./scripts/init-submodules-no-riscv-tools.sh --force
 ./scripts/init-fpga.sh
 
 if [ $1 = "group-accels" ]; then

.github/scripts/remote-run-firesim-scala-tests.sh

@@ -11,7 +11,7 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 source $SCRIPT_DIR/defaults.sh
 
 cd $REMOTE_CHIPYARD_DIR
-./scripts/init-submodules-no-riscv-tools.sh --skip-validate
+./scripts/init-submodules-no-riscv-tools.sh --force
 
 # Run Firesim Scala Tests
 export FIRESIM_ENV_SOURCED=1;

build-setup.sh

@@ -4,16 +4,12 @@
 set -e
 set -o pipefail
 
-# On macOS, use GNU readlink from 'coreutils' package in Homebrew/MacPorts
+RDIR=$(git rev-parse --show-toplevel)
-if [ "$(uname -s)" = "Darwin" ] ; then
-    READLINK=greadlink
-else
-    READLINK=readlink
-fi
 
-# If BASH_SOURCE is undefined, we may be running under zsh, in that case
+# get helpful utilities
-# provide a zsh-compatible alternative
+source $RDIR/scripts/utils.sh
-DIR="$(dirname "$($READLINK -f "${BASH_SOURCE[0]:-${(%):-%x}}")")"
+
+common_setup
 
 usage() {
     echo "Usage: ${0} [OPTIONS] [riscv-tools | esp-tools]"
@@ -28,7 +24,8 @@ usage() {
     echo "Options"
     echo "  --help -h              : Display this message"
     echo "  --unpinned-deps -ud    : Use unpinned conda environment"
-    echo "  --skip-validate        : Skip prompt checking for tagged release/conda"
+    echo "  --force -f             : Skip prompt checking for tagged release/conda"
+    echo "  --skip-validate        : DEPRECATED: Same functionality as --force"
     echo "  --skip-conda           : Skip conda env creation"
     echo "  --skip-toolchain-extra : Skip building extra RISC-V toolchain collateral (Spike, PK, tests, libgloos)"
     exit "$1"
@@ -36,7 +33,7 @@ usage() {
 
 TOOLCHAIN="riscv-tools"
 USE_PINNED_DEPS=true
-SKIP_VALIDATE_FLAG=""
+FORCE_FLAG=""
 SKIP_CONDA=false
 SKIP_TOOLCHAIN=false
 
@@ -50,8 +47,8 @@ do
             TOOLCHAIN=$1 ;;
         -ud | --unpinned-deps )
             USE_PINNED_DEPS=false ;;
-        --skip-validate)
+        --force | -f | --skip-validate)
-            SKIP_VALIDATE_FLAG=$1 ;;
+            FORCE_FLAG=$1 ;;
         --skip-conda)
             SKIP_CONDA=true ;;
         --skip-toolchain-extra)
@@ -65,8 +62,8 @@ done
 
 if [ "$SKIP_CONDA" = false ]; then
     # note: lock file must end in .conda-lock.yml - see https://github.com/conda-incubator/conda-lock/issues/154
-    LOCKFILE=$DIR/conda-requirements-$TOOLCHAIN-linux-64.conda-lock.yml
+    LOCKFILE=$RDIR/conda-requirements-$TOOLCHAIN-linux-64.conda-lock.yml
-    YAMLFILE=$DIR/conda-requirements-$TOOLCHAIN.yaml
+    YAMLFILE=$RDIR/conda-requirements-$TOOLCHAIN.yaml
 
     if [ "$USE_PINNED_DEPS" = false ]; then
         # auto-gen the lockfile
@@ -74,26 +71,28 @@ if [ "$SKIP_CONDA" = false ]; then
     fi
 
     # use conda-lock to create env
-    conda-lock install -p $DIR/.conda-env $LOCKFILE
+    conda-lock install -p $RDIR/.conda-env $LOCKFILE
 
-    source $DIR/.conda-env/etc/profile.d/conda.sh
+    source $RDIR/.conda-env/etc/profile.d/conda.sh
-    conda activate $DIR/.conda-env
+    conda activate $RDIR/.conda-env
 fi
 
-if [ -z "$SKIP_VALIDATE_FLAG" ]; then
+if [ -z "$FORCE_FLAG" ]; then
     if [ -z ${CONDA_DEFAULT_ENV+x} ]; then
         error "ERROR: No conda environment detected. Did you activate the conda environment (e.x. 'conda activate base')?"
         exit 1
     fi
 fi
 
-$DIR/scripts/init-submodules-no-riscv-tools.sh $SKIP_VALIDATE_FLAG
+$RDIR/scripts/init-submodules-no-riscv-tools.sh $FORCE_FLAG
 
 if [ "$SKIP_TOOLCHAIN" = false ]; then
-    $DIR/scripts/build-toolchain-extra.sh $SKIP_VALIDATE_FLAG $TOOLCHAIN
+    $RDIR/scripts/build-toolchain-extra.sh $FORCE_FLAG $TOOLCHAIN
 fi
 
+$RDIR/scripts/gen-tags.sh
+
 cat << EOT >> env.sh
 # line auto-generated by $0
-conda activate $DIR/.conda-env
+conda activate $RDIR/.conda-env
 EOT

scripts/build-toolchain-extra.sh

@@ -1,22 +1,15 @@
 #!/usr/bin/env bash
 
-# this script is based on the firesim build toolchains script
-
 # exit script if any command fails
 set -e
 set -o pipefail
 
-# On macOS, use GNU readlink from 'coreutils' package in Homebrew/MacPorts
+RDIR=$(git rev-parse --show-toplevel)
-if [ "$(uname -s)" = "Darwin" ] ; then
-    READLINK=greadlink
-else
-    READLINK=readlink
-fi
 
-# If BASH_SOURCE is undefined, we may be running under zsh, in that case
+# get helpful utilities
-# provide a zsh-compatible alternative
+source $RDIR/scripts/utils.sh
-DIR="$(dirname "$($READLINK -f "${BASH_SOURCE[0]:-${(%):-%x}}")")"
+
-CHIPYARD_DIR="$(dirname "$DIR")"
+common_setup
 
 # Allow user to override MAKE
 [ -n "${MAKE:+x}" ] || MAKE=$(command -v gnumake || command -v gmake || command -v make)
@@ -33,29 +26,22 @@ usage() {
     echo "   --prefix PREFIX       : Install destination. If unset, defaults to $CONDA_PREFIX/riscv-tools"
     echo "                           or $CONDA_PREFIX/esp-tools"
     echo "   --clean-after-install : Run make clean in calls to module_make and module_build"
-    echo "   --skip-validate       : Skip prompt checking for conda"
+    echo "   --force -f            : Skip prompt checking for conda"
+    echo "   --skip-validate       : DEPRECATED: Same functionality as --force"
     echo "   --help -h             : Display this message"
     exit "$1"
 }
 
-error() {
-    echo "${0##*/}: ${1}" >&2
-}
-die() {
-    error "$1"
-    exit "${2:--1}"
-}
-
 TOOLCHAIN="riscv-tools"
 CLEANAFTERINSTALL=""
 RISCV=""
-SKIP_VALIDATE=false
+FORCE=false
 
 # getopts does not support long options, and is inflexible
 while [ "$1" != "" ];
 do
     case $1 in
-        -h | --help | help )
+        -h | -H | --help | help )
             usage 3 ;;
         -p | --prefix )
             shift
@@ -64,8 +50,8 @@ do
             CLEANAFTERINSTALL="true" ;;
         riscv-tools | esp-tools)
             TOOLCHAIN=$1 ;;
-        --skip-validate)
+        --force | -f | --skip-validate)
-            SKIP_VALIDATE=true;
+            FORCE=true;
             ;;
         * )
             error "invalid option $1"
@@ -74,7 +60,7 @@ do
     shift
 done
 
-if [ "$SKIP_VALIDATE" = false ]; then
+if [ "$FORCE" = false ]; then
     if [ -z ${CONDA_DEFAULT_ENV+x} ]; then
         error "ERROR: No conda environment detected. Did you activate the conda environment (e.x. 'conda activate chipyard')?"
         exit 1
@@ -82,7 +68,7 @@ if [ "$SKIP_VALIDATE" = false ]; then
 fi
 
 if [ -z "$RISCV" ] ; then
-      RISCV="$CONDA_PREFIX/$TOOLCHAIN"
+    RISCV="$CONDA_PREFIX/$TOOLCHAIN"
 fi
 
 XLEN=64
@@ -92,7 +78,7 @@ echo "Installing extra toolchain utilities/tests to $RISCV"
 # install risc-v tools
 export RISCV="$RISCV"
 
-cd "${CHIPYARD_DIR}"
+cd "${RDIR}"
 
 SRCDIR="$(pwd)/toolchains/${TOOLCHAIN}"
 [ -d "${SRCDIR}" ] || die "unsupported toolchain: ${TOOLCHAIN}"

scripts/firesim-setup.sh

@@ -5,13 +5,12 @@
 set -e
 set -o pipefail
 
-RDIR=$(pwd)
+RDIR=$(git rev-parse --show-toplevel)
-scripts_dir="$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )"
 
-cd "${scripts_dir}/.."
+cd "$RDIR"
 
 # Reenable the FireSim submodule
 git config --unset submodule.sims/firesim.update || true
-cd sims/firesim
+pushd sims/firesim
 ./build-setup.sh "$@" --library --skip-validate
-cd "$RDIR"
+popd

scripts/init-fpga.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+
 # exit script if any command fails
 set -e
 set -o pipefail

scripts/init-software.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+
 # exit script if any command fails
 set -e
 set -o pipefail

scripts/init-submodules-no-riscv-tools-nolog.sh

@@ -4,27 +4,35 @@
 set -e
 set -o pipefail
 
-SKIP_VALIDATE=false
+RDIR=$(git rev-parse --show-toplevel)
+
+# get helpful utilities
+source $RDIR/scripts/utils.sh
+
+common_setup
 
 function usage
 {
-    echo "Usage: $0 [--skip-validate]"
+    echo "Usage: $0 [--force]"
     echo "Initialize Chipyard submodules and setup initial env.sh script."
     echo ""
-    echo "  --skip-validate    Skip prompt checking for tagged release"
+    echo "  --force -f      : Skip prompt checking for tagged release"
+    echo "  --skip-validate : DEPRECATED: Same functionality as --force"
 }
 
+FORCE=false
 while test $# -gt 0
 do
    case "$1" in
-        --skip-validate)
+        --force | -f | --skip-validate)
-            SKIP_VALIDATE=true;
+            FORCE=true;
             ;;
-        -h | -H | --help)
+        -h | -H | --help | help)
             usage
             exit 1
             ;;
-        *) echo "ERROR: bad argument $1"
+        *)
+            echo "ERROR: bad argument $1"
             usage
             exit 2
             ;;
@@ -32,47 +40,52 @@ do
     shift
 done
 
-# Check that git version is at least 1.7.8
+# check that git version is at least 1.7.8
 MYGIT=$(git --version)
 MYGIT=${MYGIT#'git version '} # Strip prefix
 case ${MYGIT} in
-[1-9]*) ;;
+    [1-9]*)
-*) echo 'warning: unknown git version' ;;
+        ;;
+    *)
+        echo "WARNING: unknown git version"
+        ;;
 esac
 MINGIT="1.8.5"
 if [ "$MINGIT" != "$(echo -e "$MINGIT\n$MYGIT" | sort -V | head -n1)" ]; then
   echo "This script requires git version $MINGIT or greater. Exiting."
-  false
+  exit 4
 fi
 
 # before doing anything verify that you are on a release branch/tag
+save_bash_options
 set +e
-tag=$(git describe --exact-match --tags)
+git_tag=$(git describe --exact-match --tags)
-tag_ret_code=$?
+git_tag_rc=$?
-set -e
+restore_bash_options
-if [ $tag_ret_code -ne 0 ]; then
+if [ "$git_tag_rc" -ne 0 ]; then
-    if [ "$SKIP_VALIDATE" = false ]; then
+    if [ "$FORCE" == false ]; then
-        read -p "WARNING: You are not on an official release of Chipyard."$'\n'"Type \"y\" to continue if this is intended, otherwise see https://chipyard.readthedocs.io/en/stable/Chipyard-Basics/Initial-Repo-Setup.html#setting-up-the-chipyard-repo: " validate
+        while true; do
-        [[ $validate == [yY] ]] || exit 3
+            read -p "WARNING: You are not on an official release of Chipyard."$'\n'"Type \"y\" to continue if this is intended or \"n\" if not: " validate
-        echo "Setting up non-official Chipyard release"
+            case "$validate" in
+                y | Y)
+                    echo "Continuing on to setting up non-official Chipyard release repository"
+                    break
+                    ;;
+                n | N)
+                    error "See https://chipyard.readthedocs.io/en/stable/Chipyard-Basics/Initial-Repo-Setup.html#setting-up-the-chipyard-repo for setting up an official release of Chipyard. "
+                    exit 3
+                    ;;
+                *)
+                    error "Invalid response. Please type \"y\" or \"n\""
+                    ;;
+            esac
+        done
     fi
 else
-    echo "Setting up official Chipyard release: $tag"
+    echo "Setting up official Chipyard release: $git_tag"
 fi
 
-# On macOS, use GNU readlink from 'coreutils' package in Homebrew/MacPorts
+cd "$RDIR"
-if [ "$(uname -s)" = "Darwin" ] ; then
-    READLINK=greadlink
-else
-    READLINK=readlink
-fi
-
-# If BASH_SOURCE is undefined we may be running under zsh, in that case
-# provide a zsh-compatible alternative
-DIR="$(dirname "$($READLINK -f "${BASH_SOURCE[0]:-${(%):-%x}}")")"
-CHIPYARD_DIR="$(dirname "$DIR")"
-
-cd "$CHIPYARD_DIR"
 
 (
     # Blocklist of submodules to initially skip:
@@ -106,29 +119,28 @@ cd "$CHIPYARD_DIR"
     _unskip() { git config --local --unset-all "submodule.${1}.update" || : ; }
 
     trap 'git_submodule_exclude _unskip' EXIT INT TERM
-    set -x
+    (
-    git_submodule_exclude _skip
+        set -x
-    git submodule update --init --recursive #--jobs 8
+        git_submodule_exclude _skip
-    set +x
+        git submodule update --init --recursive #--jobs 8
+    )
 )
 
-set -x
+(
+    # Non-recursive clone to exclude riscv-linux
+    git submodule update --init generators/sha3
 
-# Non-recursive clone to exclude riscv-linux
+    # Non-recursive clone to exclude gemmini-software
-git submodule update --init generators/sha3
+    git submodule update --init generators/gemmini
+    git -C generators/gemmini/ submodule update --init --recursive software/gemmini-rocc-tests
 
-# Non-recursive clone to exclude gemmini-software
+    # Minimal non-recursive clone to initialize sbt dependencies
-git submodule update --init generators/gemmini
+    git submodule update --init sims/firesim
-git -C generators/gemmini/ submodule update --init --recursive software/gemmini-rocc-tests
+    git config --local submodule.sims/firesim.update none
 
-# Minimal non-recursive clone to initialize sbt dependencies
+    # Only shallow clone needed for basic SW tests
-git submodule update --init sims/firesim
+    git submodule update --init software/firemarshal
-git config --local submodule.sims/firesim.update none
+)
-
-# Only shallow clone needed for basic SW tests
-git submodule update --init software/firemarshal
-
-set +x
 
 # Configure firemarshal to know where our firesim installation is
 if [ ! -f ./software/firemarshal/marshal-config.yaml ]; then

scripts/init-vlsi.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+
 # exit script if any command fails
 set -e
 set -o pipefail

scripts/repo-clean.sh

@@ -3,19 +3,9 @@
 set -e
 
 # this should be run from chipyard repo top
-TOPDIR=$(pwd)
+RDIR=$(git rev-parse --show-toplevel)
 
-cd generators/cva6/src/main/resources/vsrc
+cd $RDIR/libgloss
-git submodule deinit cva6
-
-cd $TOPDIR
-
-cd toolchains/qemu/roms/
-git submodule deinit edk2
-cd ../
-rm -rf build
-
-cd ../libgloss
 rm -rf build.log
 
 cd ../riscv-tools/riscv-isa-sim/
@@ -27,6 +17,5 @@ rm -rf build.log
 cd ../riscv-tests
 rm -rf build.log
 
-cd $TOPDIR
+cd $RDIR/tools/api-config-chipsalliance
-cd tools/api-config-chipsalliance
 git config --local status.showUntrackedFiles no

scripts/tutorial-setup.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
 
-set -e -x
+set -ex
+
+RDIR=$(git rev-parse --show-toplevel)
+
+cd $RDIR
 
 git rm generators/chipyard/src/main/scala/config/RocketSha3Configs.scala
 git rm -rf generators/sha3

scripts/utils.sh

@@ -0,0 +1,55 @@
+#/usr/bin/env bash
+
+#######################################
+# Common setup. Init MacOS compatibility
+# variables.
+# Globals:
+#   READLINK
+#######################################
+function common_setup
+{
+    # On macOS, use GNU readlink from 'coreutils' package in Homebrew/MacPorts
+    if [ "$(uname -s)" = "Darwin" ] ; then
+        READLINK=greadlink
+    else
+        READLINK=readlink
+    fi
+}
+
+#######################################
+# Error echo wrapper
+#######################################
+function error
+{
+    echo "${0##*/}: ${1}" >&2
+}
+
+#######################################
+# Error then exit wrapper
+# Arguments:
+#   string to print before exit
+#   (optional) int error code
+#######################################
+function die
+{
+    error "$1"
+    exit "${2:--1}"
+}
+
+#######################################
+# Save bash options. Must be called
+# before a corresponding `restore_bash_options`.
+#######################################
+function save_bash_options
+{
+    OLDSTATE=$(set +o)
+}
+
+#######################################
+# Restore bash options. Must be called
+# after a corresponding `save_bash_options`.
+#######################################
+function restore_bash_options
+{
+    set +vx; eval "$OLDSTATE"
+}