Use conda + Update initial setup docs

This commit is contained in:
abejgonzalez
2022-04-28 15:10:46 -07:00
committed by Abraham Gonzalez
parent 684a02a10f
commit 1de35a6af4
46 changed files with 742 additions and 841 deletions

View File

@@ -1,227 +0,0 @@
#!/usr/bin/env bash
#this script is based on the firesim build toolchains script
# exit script if any command fails
set -e
set -o pipefail
# On macOS, use GNU readlink from 'coreutils' package in Homebrew/MacPorts
if [ "$(uname -s)" = "Darwin" ] ; then
READLINK=greadlink
else
READLINK=readlink
fi
# If BASH_SOURCE is undefined, we may be running under zsh, in that case
# provide a zsh-compatible alternative
DIR="$(dirname "$($READLINK -f "${BASH_SOURCE[0]:-${(%):-%x}}")")"
CHIPYARD_DIR="$(dirname "$DIR")"
# Allow user to override MAKE
[ -n "${MAKE:+x}" ] || MAKE=$(command -v gnumake || command -v gmake || command -v make)
readonly MAKE
usage() {
echo "usage: ${0} [OPTIONS] [riscv-tools | esp-tools | ec2fast]"
echo ""
echo "Installation Types"
echo " riscv-tools: if set, builds the riscv toolchain (this is also the default)"
echo " esp-tools: if set, builds esp-tools toolchain used for the hwacha vector accelerator"
echo " ec2fast: if set, pulls in a pre-compiled RISC-V toolchain for an EC2 manager instance"
echo ""
echo "Options"
echo " --prefix PREFIX : Install destination. If unset, defaults to $(pwd)/riscv-tools-install"
echo " or $(pwd)/esp-tools-install"
echo " --ignore-qemu : Ignore installing QEMU"
echo " --clean-after-install : Run make clean in calls to module_make and module_build"
echo " --arch -a : Architecture (e.g., rv64gc)"
echo " --help -h : Display this message"
exit "$1"
}
error() {
echo "${0##*/}: ${1}" >&2
}
die() {
error "$1"
exit "${2:--1}"
}
TOOLCHAIN="riscv-tools"
EC2FASTINSTALL="false"
IGNOREQEMU=""
CLEANAFTERINSTALL=""
RISCV=""
ARCH=""
# getopts does not support long options, and is inflexible
while [ "$1" != "" ];
do
case $1 in
-h | --help | help )
usage 3 ;;
-p | --prefix )
shift
RISCV=$(realpath $1) ;;
--ignore-qemu )
IGNOREQEMU="true" ;;
-a | --arch )
shift
ARCH=$1 ;;
--clean-after-install )
CLEANAFTERINSTALL="true" ;;
riscv-tools | esp-tools)
TOOLCHAIN=$1 ;;
ec2fast )
EC2FASTINSTALL="true" ;;
* )
error "invalid option $1"
usage 1 ;;
esac
shift
done
if [ -z "$RISCV" ] ; then
INSTALL_DIR="$TOOLCHAIN-install"
RISCV="$(pwd)/$INSTALL_DIR"
fi
if [ -z "$ARCH" ] ; then
XLEN=64
elif [[ "$ARCH" =~ ^rv(32|64)((i?m?a?f?d?|g?)c?)$ ]]; then
XLEN=${BASH_REMATCH[1]}
else
error "invalid arch $ARCH"
usage 1
fi
echo "Installing toolchain to $RISCV"
# install risc-v tools
export RISCV="$RISCV"
cd "${CHIPYARD_DIR}"
SRCDIR="$(pwd)/toolchains/${TOOLCHAIN}"
[ -d "${SRCDIR}" ] || die "unsupported toolchain: ${TOOLCHAIN}"
. ./scripts/build-util.sh
if [ "${EC2FASTINSTALL}" = true ] ; then
[ "${TOOLCHAIN}" = 'riscv-tools' ] ||
die "unsupported precompiled toolchain: ${TOOLCHAIN}"
echo '=> Fetching pre-built toolchain'
module=toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt
git config --unset submodule."${module}".update || :
git submodule update --init --depth 1 "${module}"
echo '==> Verifying toolchain version hash'
# Find commit hash without initializing the submodule
hashsrc="$(git ls-tree -d HEAD "${SRCDIR}/riscv-gnu-toolchain" | {
unset IFS && read -r _ type obj _ &&
test -n "${obj}" && test "${type}" = 'commit' && echo "${obj}"
}; )" ||
die 'failed to obtain riscv-gnu-toolchain submodule hash' "$?"
read -r hashbin < "${module}/HASH" ||
die 'failed to obtain riscv-gnu-toolchain-prebuilt hash' "$?"
echo "==> ${hashsrc}"
[ "${hashsrc}" = "${hashbin}" ] ||
die "pre-built version mismatch: ${hashbin}"
echo '==> Installing pre-built toolchain'
"${MAKE}" -C "${module}" DESTDIR="${RISCV}" install
git submodule deinit "${module}" || :
else
MAKE_VER=$("${MAKE}" --version) || true
case ${MAKE_VER} in
'GNU Make '[4-9]\.*)
;;
'GNU Make '[1-9][0-9])
;;
*)
die 'obsolete make version; need GNU make 4.x or later'
;;
esac
module_prepare riscv-gnu-toolchain qemu
module_build riscv-gnu-toolchain --prefix="${RISCV}" --with-cmodel=medany ${ARCH:+--with-arch=${ARCH}}
echo '==> Building GNU/Linux toolchain'
module_make riscv-gnu-toolchain linux
fi
# disable boost explicitly for https://github.com/riscv-software-src/riscv-isa-sim/issues/834
# since we don't have it in our requirements
module_all riscv-isa-sim --prefix="${RISCV}" --with-boost=no --with-boost-asio=no --with-boost-regex=no
# build static libfesvr library for linking into firesim driver (or others)
echo '==> Installing libfesvr static library'
module_make riscv-isa-sim libfesvr.a
cp -p "${SRCDIR}/riscv-isa-sim/build/libfesvr.a" "${RISCV}/lib/"
CC= CXX= module_all riscv-pk --prefix="${RISCV}" --host=riscv${XLEN}-unknown-elf
module_all riscv-tests --prefix="${RISCV}/riscv${XLEN}-unknown-elf" --with-xlen=${XLEN}
# Common tools (not in any particular toolchain dir)
CC= CXX= SRCDIR="$(pwd)/toolchains" module_all libgloss --prefix="${RISCV}/riscv${XLEN}-unknown-elf" --host=riscv${XLEN}-unknown-elf
if [ -z "$IGNOREQEMU" ] ; then
echo "=> Starting qemu build"
dir="$(pwd)/toolchains/qemu"
echo "==> Initializing qemu submodule"
#since we don't want to use the global config we init passing rewrite config in to the command
git -c url.https://github.com/qemu.insteadOf=https://git.qemu.org/git submodule update --init --recursive "$dir"
echo "==> Applying url-rewriting to avoid git.qemu.org"
# and once the clones exist, we recurse through them and set the rewrite
# in the local config so that any further commands by the user have the rewrite. uggh. git, why you so ugly?
git -C "$dir" config --local url.https://github.com/qemu.insteadOf https://git.qemu.org/git
git -C "$dir" submodule foreach --recursive 'git config --local url.https://github.com/qemu.insteadOf https://git.qemu.org/git'
# check to see whether the rewrite rules are needed any more
# If you find git.qemu.org in any .gitmodules file below qemu, you still need them
# the /dev/null redirection in the submodule grepping is to quiet non-existance of further .gitmodules
! grep -q 'git\.qemu\.org' "$dir/.gitmodules" && \
git -C "$dir" submodule foreach --quiet --recursive '! grep -q "git\.qemu\.org" .gitmodules 2>/dev/null' && \
echo "==> PLEASE REMOVE qemu URL-REWRITING from scripts/build-toolchains.sh. It is no longer needed!" && exit 1
(
# newer version of BFD-based ld has made '-no-pie' an error because it renamed to '--no-pie'
# meanwhile, ld.gold will still accept '-no-pie'
# QEMU 5.0 still uses '-no-pie' in it's linker options
# default LD to ld if it isn't set
if ( set +o pipefail; ${LD:-ld} -no-pie |& grep 'did you mean --no-pie' >/dev/null); then
echo "==> LD doesn't like '-no-pie'"
# LD has the problem, look for ld.gold
if type ld.gold >&/dev/null; then
echo "==> Using ld.gold to link QEMU"
export LD=ld.gold
fi
fi
# now actually do the build
SRCDIR="$(pwd)/toolchains" module_build qemu --prefix="${RISCV}" --target-list=riscv${XLEN}-softmmu --disable-werror
)
fi
# make Dromajo
git submodule update --init $CHIPYARD_DIR/tools/dromajo/dromajo-src
make -C $CHIPYARD_DIR/tools/dromajo/dromajo-src/src
# create specific env.sh
cat > "$CHIPYARD_DIR/env-$TOOLCHAIN.sh" <<EOF
# auto-generated by build-toolchains.sh
export CHIPYARD_TOOLCHAIN_SOURCED=1
export RISCV=$(printf '%q' "$RISCV")
export PATH=\${RISCV}/bin:\${PATH}
export LD_LIBRARY_PATH=\${RISCV}/lib\${LD_LIBRARY_PATH:+":\${LD_LIBRARY_PATH}"}
EOF
# create general env.sh
echo "# line auto-generated by build-toolchains.sh" >> env.sh
echo "source $(printf '%q' "$CHIPYARD_DIR/env-$TOOLCHAIN.sh")" >> env.sh
echo "Toolchain Build Complete!"

View File

@@ -1,33 +0,0 @@
#!/bin/bash
set -ex
sudo yum groupinstall -y "Development tools"
sudo yum install -y gmp-devel mpfr-devel libmpc-devel zlib-devel vim git java java-devel
# Install SBT https://www.scala-sbt.org/release/docs/Installing-sbt-on-Linux.html#Red+Hat+Enterprise+Linux+and+other+RPM-based+distributions
# sudo rm -f /etc/yum.repos.d/bintray-rpm.repo
# Use rm above if sbt installed from bintray before.
curl -L https://www.scala-sbt.org/sbt-rpm.repo > sbt-rpm.repo
sudo mv sbt-rpm.repo /etc/yum.repos.d/
sudo yum install -y sbt texinfo gengetopt
sudo yum install -y expat-devel libusb1-devel ncurses-devel cmake "perl(ExtUtils::MakeMaker)"
# deps for poky
sudo yum install -y python38 patch diffstat texi2html texinfo subversion chrpath git wget
# deps for qemu
sudo yum install -y gtk3-devel
# deps for firemarshal
sudo yum install -y python38-pip python38-devel rsync libguestfs-tools makeinfo expat ctags
# Install GNU make 4.x (needed to cross-compile glibc 2.28+)
sudo yum install -y centos-release-scl
sudo yum install -y devtoolset-8-make
# install DTC
sudo yum install -y dtc
sudo yum install -y python
# install verilator
git clone http://git.veripool.org/git/verilator
cd verilator
git checkout v4.034
autoconf && ./configure && make -j$(nproc) && sudo make install

View File

@@ -7,17 +7,24 @@ AXE_DIR=$(realpath ${SCRIPT_DIR}/../tools/axe)
ROCKET_DIR=$(realpath ${SCRIPT_DIR}/../generators/rocket-chip)
TO_AXE=${ROCKET_DIR}/scripts/toaxe.py
TO_AXE_PY3=/tmp/toaxe.py
AXE=${AXE_DIR}/src/axe
AXE_SHRINK=${AXE_DIR}/src/axe-shrink.py
AXE_SHRINK_PY3=/tmp/axe-shrink.py
# TODO: convert scripts to py3 in src
2to3 $TO_AXE -o /tmp -n -w
sed -i '30d' $TO_AXE_PY3 # remove import sets
2to3 $AXE_SHRINK -o /tmp -n -w
PATH=$PATH:${AXE_DIR}/src
grep '.*:.*#.*@' $1 > /tmp/clean-trace.txt
python2 "$TO_AXE" /tmp/clean-trace.txt > /tmp/trace.axe
python "$TO_AXE_PY3" /tmp/clean-trace.txt > /tmp/trace.axe
result=$("$AXE" check wmo /tmp/trace.axe)
if [ "$result" != OK ]; then
"$AXE_SHRINK" wmo /tmp/trace.axe
"$AXE_SHRINK_PY3" wmo /tmp/trace.axe
else
echo OK
fi

View File

@@ -0,0 +1,57 @@
name: chipyard
channels:
- conda-forge
- ucb-bar
dependencies:
# firemarshal deps
- python=3.9
- rsync
- psutil
- doit=0.35.0
- gitpython
- humanfriendly
- e2fsprogs
- ctags
- bison
- flex
- expat
# current version of buildroot won't build with make 4.3 https://github.com/firesim/FireMarshal/issues/236
- make!=4.3
- pyyaml
- unzip
- readline
- coreutils
- lzop
- qemu # from ucb-bar
# doc requirements
- sphinx
- pygments
- sphinx-autobuild
- sphinx_rtd_theme
- docutils
# misc. c/c++ compilers + related
- gcc==10.* # pinned for libelf/libdwarf builds
- gxx==10.* # pinned for libelf/libdwarf builds
- conda-gcc-specs
- binutils
# rocket-chip deps
- sbt
- openjdk
- dtc
- verilator==4.034
# chipyard deps
- dromajo # from ucb-bar
# other misc. deps
- ca-certificates
- vim
- gengetopt
- cmake
- git
- wget
- sed
- autoconf

View File

@@ -76,7 +76,6 @@ cd "$CHIPYARD_DIR"
(
# Blocklist of submodules to initially skip:
# - Toolchain submodules
# - Generators with huge submodules (e.g., linux sources)
# - FireSim until explicitly requested
# - Hammer tool plugins
@@ -84,9 +83,6 @@ cd "$CHIPYARD_DIR"
# Call the given subcommand (shell function) on each submodule
# path to temporarily exclude during the recursive update
for name in \
toolchains/*-tools/*/ \
toolchains/libgloss \
toolchains/qemu \
generators/sha3 \
generators/gemmini \
sims/firesim \
@@ -136,7 +132,12 @@ if [ ! -f ./software/firemarshal/marshal-config.yaml ]; then
echo "firesim-dir: '../../sims/firesim/'" > ./software/firemarshal/marshal-config.yaml
fi
echo "# line auto-generated by init-submodules-no-riscv-tools.sh" >> env.sh
echo '__DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]:-${(%):-%x}}")")"' >> env.sh
echo "PATH=\$__DIR/bin:\$PATH" >> env.sh
echo "PATH=\$__DIR/software/firemarshal:\$PATH" >> env.sh
cat << EOT >> env.sh
# line auto-generated by init-submodules-no-riscv-tools.sh
__DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]:-${(%):-%x}}")")"
PATH=\$__DIR/bin:\$PATH
PATH=\$__DIR/software/firemarshal:\$PATH
if [ -z \${CONDA_DEFAULT_ENV+x} ]; then
echo "WARNING: No conda environment detected. Did you activate the conda environment (e.x. 'conda activate chipyard')?"
fi
EOT

178
scripts/install-conda.sh Executable file
View File

@@ -0,0 +1,178 @@
#!/bin/bash
CONDA_INSTALL_PREFIX=/opt/conda
CONDA_INSTALLER_VERSION=4.12.0-0
CONDA_INSTALLER="https://github.com/conda-forge/miniforge/releases/download/${CONDA_INSTALLER_VERSION}/Miniforge3-${CONDA_INSTALLER_VERSION}-Linux-x86_64.sh"
CONDA_CMD="conda" # some installers install mamba or micromamba
CONDA_ENV_NAME="firesim"
DRY_RUN_OPTION=""
DRY_RUN_ECHO=()
REINSTALL_CONDA=0
usage()
{
echo "Usage: $0 [options]"
echo
echo "Options:"
echo "[--help] List this help"
echo "[--prefix <prefix>] Install prefix for conda. Defaults to /opt/conda."
echo " If <prefix>/bin/conda already exists, it will be used and install is skipped."
echo "[--dry-run] Pass-through to all conda commands and only print other commands."
echo " NOTE: --dry-run will still install conda to --prefix"
echo "[--reinstall-conda] Repairs a broken base environment by reinstalling."
echo " NOTE: will only reinstall conda and exit"
echo
echo "Examples:"
echo " % $0"
echo " Install into default system-wide prefix (using sudo if needed) and add install to system-wide /etc/profile.d"
echo " % $0 --prefix ~/conda"
echo " Install into $HOME/conda"
}
while [ $# -gt 0 ]; do
case "$1" in
--help)
usage
exit 1
;;
--prefix)
shift
CONDA_INSTALL_PREFIX="$1"
shift
;;
--dry-run)
shift
DRY_RUN_OPTION="--dry-run"
DRY_RUN_ECHO=(echo "Would Run:")
;;
--reinstall-conda)
shift
REINSTALL_CONDA=1
;;
*)
echo "Invalid Argument: $1"
usage
exit 1
;;
esac
done
if [[ $REINSTALL_CONDA -eq 1 && -n "$DRY_RUN_OPTION" ]]; then
echo "::ERROR:: --dry-run and --reinstall-conda are mutually exclusive. Pick one or the other."
fi
set -ex
set -o pipefail
# uname options are not portable so do what https://www.gnu.org/software/coreutils/faq/coreutils-faq.html#uname-is-system-specific
# suggests and iteratively probe the system type
if ! type uname >&/dev/null; then
echo "::ERROR:: need 'uname' command available to determine if we support this sytem"
exit 1
fi
if [[ "$(uname)" != "Linux" ]]; then
echo "::ERROR:: $0 only supports 'Linux' not '$(uname)'"
exit 1
fi
if [[ "$(uname -mo)" != "x86_64 GNU/Linux" ]]; then
echo "::ERROR:: $0 only supports 'x86_64 GNU/Linux' not '$(uname -io)'"
exit 1
fi
if [[ ! -r /etc/os-release ]]; then
echo "::ERROR:: $0 depends on /etc/os-release for distro-specific setup and it doesn't exist here"
exit 1
fi
OS_FLAVOR=$(grep '^ID=' /etc/os-release | awk -F= '{print $2}' | tr -d '"')
OS_VERSION=$(grep '^VERSION_ID=' /etc/os-release | awk -F= '{print $2}' | tr -d '"')
# platform-specific setup
case "$OS_FLAVOR" in
ubuntu)
;;
centos)
;;
*)
echo "::ERROR:: Unknown OS flavor '$OS_FLAVOR'. Unable to do platform-specific setup."
exit 1
;;
esac
# everything else is platform-agnostic and could easily be expanded to Windows and/or OSX
SUDO=""
prefix_parent=$(dirname "$CONDA_INSTALL_PREFIX")
if [[ ! -e "$prefix_parent" ]]; then
mkdir -p "$prefix_parent" || SUDO=sudo
elif [[ ! -w "$prefix_parent" ]]; then
SUDO=sudo
fi
if [[ -n "$SUDO" ]]; then
echo "::INFO:: using 'sudo' to install conda"
# ensure files are read-execute for everyone
umask 022
fi
if [[ -n "$SUDO" || "$(id -u)" == 0 ]]; then
INSTALL_TYPE=system
else
INSTALL_TYPE=user
fi
# to enable use of sudo and avoid modifying 'secure_path' in /etc/sudoers, we specify the full path to conda
CONDA_EXE="${CONDA_INSTALL_PREFIX}/bin/$CONDA_CMD"
if [[ -x "$CONDA_EXE" && $REINSTALL_CONDA -eq 0 ]]; then
echo "::INFO:: '$CONDA_EXE' already exists, skipping conda install"
else
wget -O install_conda.sh "$CONDA_INSTALLER" || curl -fsSLo install_conda.sh "$CONDA_INSTALLER"
if [[ $REINSTALL_CONDA -eq 1 ]]; then
conda_install_extra="-u"
echo "::INFO:: RE-installing conda to '$CONDA_INSTALL_PREFIX'"
else
conda_install_extra=""
echo "::INFO:: installing conda to '$CONDA_INSTALL_PREFIX'"
fi
# -b for non-interactive install
$SUDO bash ./install_conda.sh -b -p "$CONDA_INSTALL_PREFIX" $conda_install_extra
rm ./install_conda.sh
# see https://conda-forge.org/docs/user/tipsandtricks.html#multiple-channels
# for more information on strict channel_priority
"${DRY_RUN_ECHO[@]}" $SUDO "$CONDA_EXE" config --system --set channel_priority strict
# By default, don't mess with people's PS1, I personally find it annoying
"${DRY_RUN_ECHO[@]}" $SUDO "$CONDA_EXE" config --system --set changeps1 false
# don't automatically activate the 'base' environment when intializing shells
"${DRY_RUN_ECHO[@]}" $SUDO "$CONDA_EXE" config --system --set auto_activate_base false
# don't automatically update conda to avoid https://github.com/conda-forge/conda-libmamba-solver-feedstock/issues/2
"${DRY_RUN_ECHO[@]}" $SUDO "$CONDA_EXE" config --system --set auto_update_conda false
# conda-build is a special case and must always be installed into the base environment
$SUDO "$CONDA_EXE" install $DRY_RUN_OPTION -y -n base conda-build
# conda-libmamba-solver is a special case and must always be installed into the base environment
# see https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community
$SUDO "$CONDA_EXE" install $DRY_RUN_OPTION -y -n base conda-libmamba-solver
# Use the fast solver by default
"${DRY_RUN_ECHO[@]}" $SUDO "$CONDA_EXE" config --system --set experimental_solver libmamba
conda_init_extra_args=()
if [[ "$INSTALL_TYPE" == system ]]; then
# if we're installing into a root-owned directory using sudo, or we're already root
# initialize conda in the system-wide rcfiles
conda_init_extra_args=(--no-user --system)
fi
$SUDO "${CONDA_EXE}" init $DRY_RUN_OPTION "${conda_init_extra_args[@]}" bash
if [[ $REINSTALL_CONDA -eq 1 ]]; then
echo "::INFO:: Done reinstalling conda. Exiting"
exit 0
fi
fi

View File

@@ -1,33 +0,0 @@
#!/bin/bash
set -ex
sudo apt-get install -y build-essential bison flex software-properties-common curl
sudo apt-get install -y libgmp-dev libmpfr-dev libmpc-dev zlib1g-dev vim default-jdk default-jre
# install sbt: https://www.scala-sbt.org/release/docs/Installing-sbt-on-Linux.html#Ubuntu+and+other+Debian-based+distributions
echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | sudo tee -a /etc/apt/sources.list.d/sbt.list
curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | sudo apt-key add
sudo apt-get update
sudo apt-get install -y sbt
sudo apt-get install -y texinfo gengetopt
sudo apt-get install -y libexpat1-dev libusb-dev libncurses5-dev cmake
# deps for poky
sudo apt-get install -y python3.8 patch diffstat texi2html texinfo subversion chrpath wget
# deps for qemu
sudo apt-get install -y libgtk-3-dev gettext
# deps for firemarshal
sudo apt-get install -y python3-pip python3.8-dev rsync libguestfs-tools expat ctags
# install DTC
sudo apt-get install -y device-tree-compiler
sudo apt-get install -y python
# install git >= 2.17
sudo add-apt-repository ppa:git-core/ppa -y
sudo apt-get update
sudo apt-get install git -y
# install verilator
sudo apt-get install -y autoconf
git clone http://git.veripool.org/git/verilator
cd verilator
git checkout v4.034
autoconf && ./configure && make -j$(nproc) && sudo make install