From 799ebc6aa0f66f4a94afcbe6ef9ca10b903fe803 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 21 Mar 2017 18:35:44 -0400 Subject: [PATCH] chore: publish snapshot builds to S3 (#1078) This commit makes Appveyor and Travis CI publish snapshot builds to S3 when a pull request is merged, by making use of the `publish-aws-s3` Makefile target. The changes required for such type of deployment are the followings: - Set `S3_BUCKET` to `resin-nightly-downloads` when doing snapshot builds - Add deploy sections to `.travis.yml` and `appveyor.yml` that run `make publish-aws-s3` - Don't change `PRODUCT_NAME` when doing snapshot builds (given we'll be publishing to a different S3 bucket) - Install `awscli` in Appveyor CI and Travis CI - Make GNU/Linux Docker containers inherit `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` from the environment (so `awscli` is configured correctly inside them) - Add a prefix option to `aws-s3.sh` publish script to prepend a string to the S3 path, so we can add a timestamp to more easily distinguish files inside the `resin-nightly-downloads` bucket - Print the published URL from `aws-s3.sh` for convenience purposes, so we can click it when skimming through CI builds logs - Add the `-R` and `-L` options when recursively copying `node_modules` during a snapshot build to prevent weird Appveyor errors related to hard links. The options listed before make sure that we recursively resolve every link while copying - Move from `wget` to `curl` to avoid certificate check failures Signed-off-by: Juan Cruz Viotti --- .travis.yml | 31 +++++++++++-------- Makefile | 23 ++++++++------ appveyor.yml | 5 ++- docs/PUBLISHING.md | 2 +- docs/RUNNING-LOCALLY.md | 2 +- scripts/build/docker/Dockerfile-i686 | 4 +-- scripts/build/docker/Dockerfile-x86_64 | 4 +-- scripts/build/docker/Dockerfile.template | 4 +-- scripts/build/docker/run-command.sh | 4 +++ scripts/build/download-tool.sh | 5 +-- scripts/build/electron-download-package.sh | 2 +- .../build/electron-installer-nsis-win32.sh | 1 - scripts/publish/aws-s3.sh | 18 +++++++++-- 13 files changed, 68 insertions(+), 37 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1c8e9de2..82db4c16 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,28 +33,33 @@ before_install: fi install: - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - pip install codespell==1.9.2; - npm install -g bower; - brew install afsctool; - brew install jq; - fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + ./scripts/build/docker/run-command.sh -r ${TARGET_ARCH} -s ${PWD} -c "make info && make electron-develop"; + else + pip install codespell==1.9.2 awscli; + npm install -g bower asar; + brew install afsctool jq; make info; make electron-develop; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - ./scripts/build/docker/run-command.sh -r ${TARGET_ARCH} -s ${PWD} -c "make info && make electron-develop"; - fi script: - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - make sanity-checks && npm test; - fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./scripts/build/docker/run-command.sh -r ${TARGET_ARCH} -s ${PWD} -c "make sanity-checks && xvfb-run --server-args=$XVFB_ARGS npm test"; + else + make sanity-checks && npm test; fi +deploy: + provider: script + script: if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + ./scripts/build/docker/run-command.sh -r ${TARGET_ARCH} -s ${PWD} -c "make publish-aws-s3"; + else + make publish-aws-s3; + fi + on: + branch: master + notifications: email: false webhooks: diff --git a/Makefile b/Makefile index 41b754ef..6ac247f4 100644 --- a/Makefile +++ b/Makefile @@ -26,17 +26,18 @@ APPLICATION_COPYRIGHT = $(shell jq -r '.copyright' package.json) APPLICATION_CATEGORY = public.app-category.developer-tools APPLICATION_BUNDLE_ID = io.resin.etcher APPLICATION_FILES = lib,assets -S3_BUCKET = resin-production-downloads # Add the current commit to the version if release type is "snapshot" RELEASE_TYPE ?= snapshot PACKAGE_JSON_VERSION = $(shell jq -r '.version' package.json) ifeq ($(RELEASE_TYPE),production) APPLICATION_VERSION = $(PACKAGE_JSON_VERSION) +S3_BUCKET = resin-production-downloads endif ifeq ($(RELEASE_TYPE),snapshot) CURRENT_COMMIT_HASH = $(shell git log -1 --format="%h") APPLICATION_VERSION = $(PACKAGE_JSON_VERSION)+$(CURRENT_COMMIT_HASH) +S3_BUCKET = resin-nightly-downloads endif ifndef APPLICATION_VERSION $(error Invalid release type: $(RELEASE_TYPE)) @@ -139,13 +140,7 @@ endif TARGET_ARCH_DEBIAN = $(shell ./scripts/build/architecture-convert.sh -r $(TARGET_ARCH) -t debian) -ifeq ($(RELEASE_TYPE),production) - PRODUCT_NAME = etcher -endif -ifeq ($(RELEASE_TYPE),snapshot) - PRODUCT_NAME = etcher-snapshots -endif - +PRODUCT_NAME = etcher APPLICATION_NAME_LOWERCASE = $(shell echo $(APPLICATION_NAME) | tr A-Z a-z) APPLICATION_VERSION_DEBIAN = $(shell echo $(APPLICATION_VERSION) | tr "-" "~") @@ -192,7 +187,7 @@ $(BUILD_DIRECTORY)/electron-$(TARGET_PLATFORM)-$(APPLICATION_VERSION)-$(TARGET_A ./scripts/build/electron-create-resources-app.sh -s . -o $@ \ -v $(APPLICATION_VERSION) \ -f "$(APPLICATION_FILES)" - $(foreach prerequisite,$^,$(call execute-command,cp -rf $(prerequisite) $@)) + $(foreach prerequisite,$^,$(call execute-command,cp -RLf $(prerequisite) $@)) $(BUILD_DIRECTORY)/electron-$(TARGET_PLATFORM)-$(APPLICATION_VERSION)-$(TARGET_ARCH)-app.asar: \ $(BUILD_DIRECTORY)/electron-$(TARGET_PLATFORM)-$(APPLICATION_VERSION)-$(TARGET_ARCH)-app \ @@ -373,11 +368,21 @@ endif ifdef PUBLISH_AWS_S3 publish-aws-s3: $(PUBLISH_AWS_S3) +ifeq ($(RELEASE_TYPE),production) $(foreach publishable,$^,$(call execute-command,./scripts/publish/aws-s3.sh \ -f $(publishable) \ -b $(S3_BUCKET) \ -v $(APPLICATION_VERSION) \ -p $(PRODUCT_NAME))) +endif +ifeq ($(RELEASE_TYPE),snapshot) + $(foreach publishable,$^,$(call execute-command,./scripts/publish/aws-s3.sh \ + -f $(publishable) \ + -b $(S3_BUCKET) \ + -v $(APPLICATION_VERSION) \ + -p $(PRODUCT_NAME) \ + -k $(shell date +"%Y-%m-%d"))) +endif TARGETS += publish-aws-s3 endif diff --git a/appveyor.yml b/appveyor.yml index 556647d2..169f3c54 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,7 +28,7 @@ install: - set PATH=C:\Program Files (x86)\NSIS;%PATH% - set PATH=C:\MinGW\bin;%PATH% - set PATH=C:\MinGW\msys\1.0\bin;%PATH% - - pip install codespell==1.9.2 + - pip install codespell==1.9.2 awscli - make info - make electron-develop @@ -40,6 +40,9 @@ test_script: - make sanity-checks - cmd: npm test +deploy_script: + - cmd: IF %APPVEYOR_REPO_BRANCH%==master (make publish-aws-s3) + notifications: - provider: Webhook diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index 32f379cd..1f20e164 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -93,7 +93,7 @@ access resin.io's production downloads S3 bucket. Run the following command to publish a specific file: ```sh -./scripts/publish/aws-s3.sh -f -b -v -t +./scripts/publish/aws-s3.sh -f -b -v -p ``` Or run the following command to publish all files for the current combination diff --git a/docs/RUNNING-LOCALLY.md b/docs/RUNNING-LOCALLY.md index 91b912fd..5dd5bf23 100644 --- a/docs/RUNNING-LOCALLY.md +++ b/docs/RUNNING-LOCALLY.md @@ -33,7 +33,7 @@ The following MinGW packages are required: - `msys-make` - `msys-unzip` - `msys-zip` -- `msys-wget` +- `msys-curl` - `msys-bash` - `msys-coreutils` diff --git a/scripts/build/docker/Dockerfile-i686 b/scripts/build/docker/Dockerfile-i686 index 73d7cad3..d86a3248 100644 --- a/scripts/build/docker/Dockerfile-i686 +++ b/scripts/build/docker/Dockerfile-i686 @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y \ python-pip \ python-dev \ unzip \ - wget \ + curl \ xvfb \ zip @@ -28,4 +28,4 @@ RUN npm config set spin=false RUN npm install -g bower asar electron-installer-debian # Python -RUN pip install codespell==1.9.2 +RUN pip install codespell==1.9.2 awscli diff --git a/scripts/build/docker/Dockerfile-x86_64 b/scripts/build/docker/Dockerfile-x86_64 index 1627f792..855738a1 100644 --- a/scripts/build/docker/Dockerfile-x86_64 +++ b/scripts/build/docker/Dockerfile-x86_64 @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y \ python-pip \ python-dev \ unzip \ - wget \ + curl \ xvfb \ zip @@ -28,4 +28,4 @@ RUN npm config set spin=false RUN npm install -g bower asar electron-installer-debian # Python -RUN pip install codespell==1.9.2 +RUN pip install codespell==1.9.2 awscli diff --git a/scripts/build/docker/Dockerfile.template b/scripts/build/docker/Dockerfile.template index 4d4cedd9..d68b299f 100644 --- a/scripts/build/docker/Dockerfile.template +++ b/scripts/build/docker/Dockerfile.template @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y \ python-pip \ python-dev \ unzip \ - wget \ + curl \ xvfb \ zip @@ -28,4 +28,4 @@ RUN npm config set spin=false RUN npm install -g bower asar electron-installer-debian # Python -RUN pip install codespell==1.9.2 +RUN pip install codespell==1.9.2 awscli diff --git a/scripts/build/docker/run-command.sh b/scripts/build/docker/run-command.sh index 1c0bdb7f..1beb1734 100755 --- a/scripts/build/docker/run-command.sh +++ b/scripts/build/docker/run-command.sh @@ -82,6 +82,10 @@ fi # The `-t` and TERM setup is needed to display coloured output. docker run -t \ --env "TERM=xterm-256color" \ + --env "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" \ + --env "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" \ + --env "RELEASE_TYPE=$RELEASE_TYPE" \ + --env "CI=$CI" \ --cap-add SYS_ADMIN \ --device /dev/fuse:/dev/fuse:mrw \ --volume "$ARGV_SOURCE_CODE_DIRECTORY:/etcher" \ diff --git a/scripts/build/download-tool.sh b/scripts/build/download-tool.sh index 397e5c1e..a021b03a 100755 --- a/scripts/build/download-tool.sh +++ b/scripts/build/download-tool.sh @@ -19,7 +19,7 @@ set -u set -e -./scripts/build/check-dependency.sh wget +./scripts/build/check-dependency.sh curl SHA256SUM=$(./scripts/build/check-dependency.sh sha256sum "shasum -a 256") @@ -78,7 +78,8 @@ if [ -f "$ARGV_OUTPUT" ]; then fi fi -wget --no-check-certificate "$ARGV_URL" -O "$TEMP_OUTPUT" +echo "Downloading $ARGV_URL" +curl --continue-at - --retry 100 --location --output "$TEMP_OUTPUT" "$ARGV_URL" if ! checksum_matches "$TEMP_OUTPUT" "$ARGV_CHECKSUM"; then echo "Checksum mismatch" 1>&2 diff --git a/scripts/build/electron-download-package.sh b/scripts/build/electron-download-package.sh index 2ced0a56..71f869a1 100755 --- a/scripts/build/electron-download-package.sh +++ b/scripts/build/electron-download-package.sh @@ -60,7 +60,7 @@ ELECTRON_ARCHITECTURE=$(./scripts/build/architecture-convert.sh -r "$ARGV_ARCHIT ELECTRON_GITHUB_REPOSITORY=https://github.com/electron/electron ELECTRON_DOWNLOADS_BASEURL="$ELECTRON_GITHUB_REPOSITORY/releases/download/v$ARGV_ELECTRON_VERSION" ELECTRON_FILENAME="electron-v$ARGV_ELECTRON_VERSION-$ARGV_OPERATING_SYSTEM-$ELECTRON_ARCHITECTURE.zip" -ELECTRON_CHECKSUM=$(wget --no-check-certificate -O - "$ELECTRON_DOWNLOADS_BASEURL/SHASUMS256.txt" | grep "$ELECTRON_FILENAME" | cut -d ' ' -f 1) +ELECTRON_CHECKSUM=$(curl --location "$ELECTRON_DOWNLOADS_BASEURL/SHASUMS256.txt" | grep "$ELECTRON_FILENAME" | cut -d ' ' -f 1) ./scripts/build/download-tool.sh \ -u "$ELECTRON_DOWNLOADS_BASEURL/$ELECTRON_FILENAME" \ diff --git a/scripts/build/electron-installer-nsis-win32.sh b/scripts/build/electron-installer-nsis-win32.sh index 26292e75..7e2af035 100755 --- a/scripts/build/electron-installer-nsis-win32.sh +++ b/scripts/build/electron-installer-nsis-win32.sh @@ -18,7 +18,6 @@ set -u set -e -set -x ./scripts/build/check-dependency.sh zip diff --git a/scripts/publish/aws-s3.sh b/scripts/publish/aws-s3.sh index 5bda286a..055cc87f 100755 --- a/scripts/publish/aws-s3.sh +++ b/scripts/publish/aws-s3.sh @@ -31,6 +31,7 @@ function usage() { echo " -b " echo " -v " echo " -p " + echo " -k [S3 key prefix]" exit 1 } @@ -38,13 +39,15 @@ ARGV_FILE="" ARGV_BUCKET="" ARGV_VERSION="" ARGV_PRODUCT_NAME="" +ARGV_PREFIX="" -while getopts ":f:b:v:p:" option; do +while getopts ":f:b:v:p:k:" option; do case $option in f) ARGV_FILE="$OPTARG" ;; b) ARGV_BUCKET="$OPTARG" ;; v) ARGV_VERSION="$OPTARG" ;; p) ARGV_PRODUCT_NAME="$OPTARG" ;; + k) ARGV_PREFIX="$OPTARG" ;; *) usage ;; esac done @@ -59,8 +62,19 @@ fi FILENAME=$(basename "$ARGV_FILE") +if [ -n "$ARGV_PREFIX" ]; then + S3_KEY="$ARGV_PRODUCT_NAME/$ARGV_PREFIX/$ARGV_VERSION/$FILENAME" +else + S3_KEY="$ARGV_PRODUCT_NAME/$ARGV_VERSION/$FILENAME" +fi + aws s3api put-object \ --bucket "$ARGV_BUCKET" \ --acl public-read \ - --key "$ARGV_PRODUCT_NAME/$ARGV_VERSION/$FILENAME" \ + --key "$S3_KEY" \ --body "$ARGV_FILE" + +# Escape plus signs when printing the final URL +URL="$(echo "https://$ARGV_BUCKET.s3.amazonaws.com/$S3_KEY" | sed 's/\+/%2B/g')" + +echo "Uploaded $(basename "$ARGV_FILE") to $URL"