From a6b8301f0fa14eea50ad4979fb768c2bf11e8e80 Mon Sep 17 00:00:00 2001
From: Albert Ou <aou@eecs.berkeley.edu>
Date: Thu, 26 Sep 2019 23:04:34 -0700
Subject: [PATCH] ci: Increment toolchain cache keys to clear cache

Generate hashfiles from normalized output of git-submodule status.
(Interestingly, the old method also worked after submodule flattening.)
---
 .circleci/config.yml     | 46 ++++++++++++++++++++--------------------
 .circleci/create-hash.sh | 13 ++++++------
 2 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 855d7aba..d0453176 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -31,14 +31,14 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - run:
                 name: Building riscv-tools toolchain
                 command: |
                     .circleci/build-toolchains.sh riscv-tools
                 no_output_timeout: 120m
             - save_cache:
-                key: riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                key: riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
                 paths:
                     - "/home/riscvuser/riscv-tools-install"
     install-esp-toolchain:
@@ -55,14 +55,14 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - esp-tools-installed-v1-{{ checksum "../esp-tools.hash" }}
+                    - esp-tools-installed-v2-{{ checksum "../esp-tools.hash" }}
             - run:
                 name: Building esp-tools toolchain
                 command: |
                     .circleci/build-toolchains.sh esp-tools
                 no_output_timeout: 120m
             - save_cache:
-                key: esp-tools-installed-v1-{{ checksum "../esp-tools.hash" }}
+                key: esp-tools-installed-v2-{{ checksum "../esp-tools.hash" }}
                 paths:
                     - "/home/riscvuser/esp-tools-install"
     install-verilator:
@@ -105,7 +105,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - run:
                 name: Build extra tests
                 command: .circleci/build-extra-tests.sh
@@ -131,7 +131,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }}
@@ -160,7 +160,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }}
@@ -189,7 +189,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }}
@@ -218,7 +218,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }}
@@ -247,7 +247,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }}
@@ -276,7 +276,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - esp-tools-installed-v1-{{ checksum "../esp-tools.hash" }}
+                    - esp-tools-installed-v2-{{ checksum "../esp-tools.hash" }}
             - restore_cache:
                 keys:
                     - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }}
@@ -305,7 +305,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }}
@@ -334,7 +334,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }}
@@ -363,7 +363,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }}
@@ -389,7 +389,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }}
@@ -410,7 +410,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - example-{{ .Branch }}-{{ .Revision }}
@@ -434,7 +434,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - boomrocketexample-{{ .Branch }}-{{ .Revision }}
@@ -458,7 +458,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - boom-{{ .Branch }}-{{ .Revision }}
@@ -482,7 +482,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - rocketchip-{{ .Branch }}-{{ .Revision }}
@@ -506,7 +506,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - esp-tools-installed-v1-{{ checksum "../esp-tools.hash" }}
+                    - esp-tools-installed-v2-{{ checksum "../esp-tools.hash" }}
             - restore_cache:
                 keys:
                     - hwacha-{{ .Branch }}-{{ .Revision }}
@@ -530,7 +530,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - firesim-{{ .Branch }}-{{ .Revision }}
@@ -557,7 +557,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - fireboom-{{ .Branch }}-{{ .Revision }}
@@ -585,7 +585,7 @@ jobs:
                     .circleci/create-hash.sh
             - restore_cache:
                 keys:
-                    - riscv-tools-installed-v1-{{ checksum "../riscv-tools.hash" }}
+                    - riscv-tools-installed-v2-{{ checksum "../riscv-tools.hash" }}
             - restore_cache:
                 keys:
                     - firesim-clockdiv-{{ .Branch }}-{{ .Revision }}
diff --git a/.circleci/create-hash.sh b/.circleci/create-hash.sh
index 84a75244..63dfa242 100755
--- a/.circleci/create-hash.sh
+++ b/.circleci/create-hash.sh
@@ -4,6 +4,7 @@
 
 # turn echo on and error on earliest command
 set -ex
+set -o pipefail
 
 # get shared variables
 SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
@@ -12,10 +13,10 @@ source $SCRIPT_DIR/defaults.sh
 # enter bhd repo
 cd $LOCAL_CHIPYARD_DIR
 
-# get the version of riscv-tools from the git submodule hash
-git submodule status | grep "riscv-tools" | awk '{print$1}' | grep -o "[[:alnum:]]*" >> $HOME/riscv-tools.hash
-git submodule status | grep "esp-tools" | awk '{print$1}' | grep -o "[[:alnum:]]*" >> $HOME/esp-tools.hash
-
+# Use normalized output of git-submodule status as hashfile
+for tools in 'riscv-tools' 'esp-tools' ; do
+    git submodule status "toolchains/${tools}" | while read -r line ; do
+        echo "${line#[!0-9a-f]}"
+    done > "${HOME}/${tools}.hash"
+done
 echo "Hashfile for riscv-tools and esp-tools created in $HOME"
-echo "Contents: riscv-tools:$(cat $HOME/riscv-tools.hash)"
-echo "Contents: esp-tools:$(cat $HOME/esp-tools.hash)"