From b7a2b25eb211cce14351b8c61dead81b26b9fba1 Mon Sep 17 00:00:00 2001 From: Bryan Richter Date: Mon, 29 Jan 2024 18:58:23 +0200 Subject: [PATCH 1/4] Make USERID and HOME on the container overrideable --- automated/build.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/automated/build.sh b/automated/build.sh index b529e3f6..2d5d11ef 100755 --- a/automated/build.sh +++ b/automated/build.sh @@ -5,6 +5,12 @@ set -eu +x -o pipefail ROOT=$(cd $(dirname $0) ; pwd) TARGET=$1 +# Home on the container +: ${C_HOME:=$HOME} + +# User to run as on the container +: ${USERID:=$(id -u)} + source work/aws.sh # For nightly-YYYY-MM-DD, tag should be nightly @@ -31,7 +37,6 @@ STACK_DIR=$ROOT/work/stack DOT_STACKAGE_DIR=$ROOT/work/dot-stackage # ssh key is used for committing snapshots (and their constraints) to Github SSH_DIR=$ROOT/work/ssh -USERID=$(id -u) mkdir -p \ "$PANTRY_DIR" \ @@ -85,12 +90,12 @@ docker run --rm -v $(pwd)/stack:/exe $IMAGE /exe --version # We share pantry directory between snapshots while the other content in .stack # is stored separately (because e.g. Ubuntu releases between LTS and nightly # could differ). Also the order of binds is important. -ARGS_COMMON="--rm -v $WORKDIR:$HOME/work -w $HOME/work -v $BINDIR/curator:/usr/bin/curator:ro -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v $BINDIR/stack:/usr/bin/stack:ro -v $STACK_DIR:$HOME/.stack -v $PANTRY_DIR:$HOME/.stack/pantry" -ARGS_PREBUILD="$ARGS_COMMON -u $USERID -e HOME=$HOME -v $DOT_STACKAGE_DIR:$HOME/.stackage" +ARGS_COMMON="--rm -v $WORKDIR:$C_HOME/work -w $C_HOME/work -v $BINDIR/curator:/usr/bin/curator:ro -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v $BINDIR/stack:/usr/bin/stack:ro -v $STACK_DIR:$C_HOME/.stack -v $PANTRY_DIR:$C_HOME/.stack/pantry" +ARGS_PREBUILD="$ARGS_COMMON -u $USERID -e HOME=$C_HOME -v $DOT_STACKAGE_DIR:$C_HOME/.stackage" ARGS_BUILD="$ARGS_COMMON" # instance-data is an undocumented feature of S3 used by amazonka, # see https://github.com/brendanhay/amazonka/issues/271 -ARGS_UPLOAD="$ARGS_COMMON -u $USERID -e HOME=$HOME -v $HACKAGE_CREDS:/hackage-creds:ro -v $DOT_STACKAGE_DIR:$HOME/.stackage -v $SSH_DIR:$HOME/.ssh:ro -v $GITCONFIG:$HOME/.gitconfig:ro -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -v $DOT_STACKAGE_DIR:/dot-stackage" +ARGS_UPLOAD="$ARGS_COMMON -u $USERID -e HOME=$C_HOME -v $HACKAGE_CREDS:/hackage-creds:ro -v $DOT_STACKAGE_DIR:$C_HOME/.stackage -v $SSH_DIR:$C_HOME/.ssh:ro -v $GITCONFIG:$C_HOME/.gitconfig:ro -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -v $DOT_STACKAGE_DIR:/dot-stackage" # Make sure we actually need this snapshot. We only check this for LTS releases # since, for nightlies, we'd like to run builds even if they are unnecessary to From 56aab7119e756d4dd3c84627884ef4e77421d9dc Mon Sep 17 00:00:00 2001 From: Bryan Richter Date: Tue, 30 Jan 2024 10:47:39 +0200 Subject: [PATCH 2/4] Make empty NOPLAN safe --- automated/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated/build.sh b/automated/build.sh index 2d5d11ef..9bc055a8 100755 --- a/automated/build.sh +++ b/automated/build.sh @@ -20,7 +20,7 @@ if [ $SHORTNAME = "lts" ] then TAG=$(echo $TARGET | sed 's@^lts-\([0-9]*\)\.[0-9]*@lts\1@') WORKDIR=$ROOT/work/$(echo $TARGET | sed 's@^lts-\([0-9]*\)\.[0-9]*@lts-\1@') - if [ -n "$NOPLAN" ]; then + if [ -n "${NOPLAN:-}" ]; then echo '* DO NOT EDIT work/ files: commit to lts-haskell/build-constraints! *' exit 1 fi From 91685e52e45fb00f14d977349a82c4447e6e9cdd Mon Sep 17 00:00:00 2001 From: Bryan Richter Date: Tue, 30 Jan 2024 10:48:04 +0200 Subject: [PATCH 3/4] Update to curator that supports R2 --- automated/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated/build.sh b/automated/build.sh index 9bc055a8..684f20ec 100755 --- a/automated/build.sh +++ b/automated/build.sh @@ -73,7 +73,7 @@ BINDIR=$(cd $ROOT/work/bin ; pwd) cd $BINDIR rm -f curator stack *.bz2 -curl -L "https://github.com/commercialhaskell/curator/releases/download/commit-b1528dc5eefd100bc9f98dee108f8ad8c8cb4006/curator.bz2" | bunzip2 > curator +curl -L "https://github.com/commercialhaskell/curator/releases/download/commit-dc6e10c5f2144b36794917b512cff13ac5979ff3/curator.bz2" | bunzip2 > curator chmod +x curator echo -n "curator version: " docker run --rm -v $(pwd)/curator:/exe $IMAGE /exe --version From 8f8fc126b1febceab58d81a81be1dc2e8d4f599b Mon Sep 17 00:00:00 2001 From: Bryan Richter Date: Tue, 30 Jan 2024 11:27:41 +0200 Subject: [PATCH 4/4] Enable overriding bucket name and AWS_ENDPOINT_URL No change in behavior if they are unset. --- automated/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated/build.sh b/automated/build.sh index 684f20ec..76a926d1 100755 --- a/automated/build.sh +++ b/automated/build.sh @@ -95,7 +95,7 @@ ARGS_PREBUILD="$ARGS_COMMON -u $USERID -e HOME=$C_HOME -v $DOT_STACKAGE_DIR:$C_H ARGS_BUILD="$ARGS_COMMON" # instance-data is an undocumented feature of S3 used by amazonka, # see https://github.com/brendanhay/amazonka/issues/271 -ARGS_UPLOAD="$ARGS_COMMON -u $USERID -e HOME=$C_HOME -v $HACKAGE_CREDS:/hackage-creds:ro -v $DOT_STACKAGE_DIR:$C_HOME/.stackage -v $SSH_DIR:$C_HOME/.ssh:ro -v $GITCONFIG:$C_HOME/.gitconfig:ro -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -v $DOT_STACKAGE_DIR:/dot-stackage" +ARGS_UPLOAD="$ARGS_COMMON -u $USERID -e HOME=$C_HOME -v $HACKAGE_CREDS:/hackage-creds:ro -v $DOT_STACKAGE_DIR:$C_HOME/.stackage -v $SSH_DIR:$C_HOME/.ssh:ro -v $GITCONFIG:$C_HOME/.gitconfig:ro -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY ${AWS_ENDPOINT_URL:+-e AWS_ENDPOINT_URL=$AWS_ENDPOINT_URL} -v $DOT_STACKAGE_DIR:/dot-stackage" # Make sure we actually need this snapshot. We only check this for LTS releases # since, for nightlies, we'd like to run builds even if they are unnecessary to @@ -151,7 +151,7 @@ docker run $ARGS_UPLOAD $IMAGE /bin/bash -c "exec curator check-target-available # # * Upload the docs to S3 # * Upload the new snapshot .yaml file to the appropriate Github repo, also upload its constraints -docker run $ARGS_UPLOAD $IMAGE /bin/bash -c "curator upload-docs --target $TARGET && curator upload-github --target $TARGET" +docker run $ARGS_UPLOAD $IMAGE /bin/bash -c "curator upload-docs --target $TARGET ${DOCS_BUCKET:+--bucket $DOCS_BUCKET}" && curator upload-github --target $TARGET" # fixed in https://github.com/commercialhaskell/curator/pull/24 docker run $ARGS_UPLOAD $IMAGE /bin/bash -c "exec curator hackage-distro --target $TARGET"