chore: publish snapshot builds to S3 (#1078)

This commit makes Appveyor and Travis CI publish snapshot builds to S3
when a pull request is merged, by making use of the `publish-aws-s3`
Makefile target.

The changes required for such type of deployment are the followings:

- Set `S3_BUCKET` to `resin-nightly-downloads` when doing snapshot
  builds

- Add deploy sections to `.travis.yml` and `appveyor.yml` that run `make
  publish-aws-s3`

- Don't change `PRODUCT_NAME` when doing snapshot builds (given we'll be
  publishing to a different S3 bucket)

- Install `awscli` in Appveyor CI and Travis CI

- Make GNU/Linux Docker containers inherit `AWS_ACCESS_KEY_ID` and
  `AWS_SECRET_ACCESS_KEY` from the environment (so `awscli` is
  configured correctly inside them)

- Add a prefix option to `aws-s3.sh` publish script to prepend a string
  to the S3 path, so we can add a timestamp to more easily distinguish
  files inside the `resin-nightly-downloads` bucket

- Print the published URL from `aws-s3.sh` for convenience purposes, so
  we can click it when skimming through CI builds logs

- Add the `-R` and `-L` options when recursively copying `node_modules`
  during a snapshot build to prevent weird Appveyor errors related to
  hard links. The options listed before make sure that we recursively
  resolve every link while copying

- Move from `wget` to `curl` to avoid certificate check failures

Signed-off-by: Juan Cruz Viotti <jviotti@openmailbox.org>

.travis.yml

@@ -33,28 +33,33 @@ before_install:
     fi
 
 install:
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      pip install codespell==1.9.2;
+      ./scripts/build/docker/run-command.sh -r ${TARGET_ARCH} -s ${PWD} -c "make info && make electron-develop";
-      npm install -g bower;
+    else
-      brew install afsctool;
+      pip install codespell==1.9.2 awscli;
-      brew install jq;
+      npm install -g bower asar;
-    fi
+      brew install afsctool jq;
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       make info;
       make electron-develop;
     fi
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      ./scripts/build/docker/run-command.sh -r ${TARGET_ARCH} -s ${PWD} -c "make info && make electron-develop";
-    fi
 
 script:
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      make sanity-checks && npm test;
-    fi
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
       ./scripts/build/docker/run-command.sh -r ${TARGET_ARCH} -s ${PWD} -c "make sanity-checks && xvfb-run --server-args=$XVFB_ARGS npm test";
+    else
+      make sanity-checks && npm test;
     fi
 
+deploy:
+  provider: script
+  script: if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+            ./scripts/build/docker/run-command.sh -r ${TARGET_ARCH} -s ${PWD} -c "make publish-aws-s3";
+          else
+            make publish-aws-s3;
+          fi
+  on:
+    branch: master
+
 notifications:
   email: false
   webhooks:

Makefile

@@ -26,17 +26,18 @@ APPLICATION_COPYRIGHT = $(shell jq -r '.copyright' package.json)
 APPLICATION_CATEGORY = public.app-category.developer-tools
 APPLICATION_BUNDLE_ID = io.resin.etcher
 APPLICATION_FILES = lib,assets
-S3_BUCKET = resin-production-downloads
 
 # Add the current commit to the version if release type is "snapshot"
 RELEASE_TYPE ?= snapshot
 PACKAGE_JSON_VERSION = $(shell jq -r '.version' package.json)
 ifeq ($(RELEASE_TYPE),production)
 APPLICATION_VERSION = $(PACKAGE_JSON_VERSION)
+S3_BUCKET = resin-production-downloads
 endif
 ifeq ($(RELEASE_TYPE),snapshot)
 CURRENT_COMMIT_HASH = $(shell git log -1 --format="%h")
 APPLICATION_VERSION = $(PACKAGE_JSON_VERSION)+$(CURRENT_COMMIT_HASH)
+S3_BUCKET = resin-nightly-downloads
 endif
 ifndef APPLICATION_VERSION
 $(error Invalid release type: $(RELEASE_TYPE))
@@ -139,13 +140,7 @@ endif
 
 TARGET_ARCH_DEBIAN = $(shell ./scripts/build/architecture-convert.sh -r $(TARGET_ARCH) -t debian)
 
-ifeq ($(RELEASE_TYPE),production)
+PRODUCT_NAME = etcher
-	PRODUCT_NAME = etcher
-endif
-ifeq ($(RELEASE_TYPE),snapshot)
-	PRODUCT_NAME = etcher-snapshots
-endif
-
 APPLICATION_NAME_LOWERCASE = $(shell echo $(APPLICATION_NAME) | tr A-Z a-z)
 APPLICATION_VERSION_DEBIAN = $(shell echo $(APPLICATION_VERSION) | tr "-" "~")
 
@@ -192,7 +187,7 @@ $(BUILD_DIRECTORY)/electron-$(TARGET_PLATFORM)-$(APPLICATION_VERSION)-$(TARGET_A
 	./scripts/build/electron-create-resources-app.sh -s . -o $@ \
 		-v $(APPLICATION_VERSION) \
 		-f "$(APPLICATION_FILES)"
-	$(foreach prerequisite,$^,$(call execute-command,cp -rf $(prerequisite) $@))
+	$(foreach prerequisite,$^,$(call execute-command,cp -RLf $(prerequisite) $@))
 
 $(BUILD_DIRECTORY)/electron-$(TARGET_PLATFORM)-$(APPLICATION_VERSION)-$(TARGET_ARCH)-app.asar: \
 	$(BUILD_DIRECTORY)/electron-$(TARGET_PLATFORM)-$(APPLICATION_VERSION)-$(TARGET_ARCH)-app \
@@ -373,11 +368,21 @@ endif
 
 ifdef PUBLISH_AWS_S3
 publish-aws-s3: $(PUBLISH_AWS_S3)
+ifeq ($(RELEASE_TYPE),production)
 	$(foreach publishable,$^,$(call execute-command,./scripts/publish/aws-s3.sh \
 		-f $(publishable) \
 		-b $(S3_BUCKET) \
 		-v $(APPLICATION_VERSION) \
 		-p $(PRODUCT_NAME)))
+endif
+ifeq ($(RELEASE_TYPE),snapshot)
+	$(foreach publishable,$^,$(call execute-command,./scripts/publish/aws-s3.sh \
+		-f $(publishable) \
+		-b $(S3_BUCKET) \
+		-v $(APPLICATION_VERSION) \
+		-p $(PRODUCT_NAME) \
+		-k $(shell date +"%Y-%m-%d")))
+endif
 
 TARGETS += publish-aws-s3
 endif

appveyor.yml

@@ -28,7 +28,7 @@ install:
   - set PATH=C:\Program Files (x86)\NSIS;%PATH%
   - set PATH=C:\MinGW\bin;%PATH%
   - set PATH=C:\MinGW\msys\1.0\bin;%PATH%
-  - pip install codespell==1.9.2
+  - pip install codespell==1.9.2 awscli
   - make info
   - make electron-develop
 
@@ -40,6 +40,9 @@ test_script:
   - make sanity-checks
   - cmd: npm test
 
+deploy_script:
+  - cmd: IF %APPVEYOR_REPO_BRANCH%==master (make publish-aws-s3)
+
 notifications:
 
   - provider: Webhook

docs/PUBLISHING.md

@@ -93,7 +93,7 @@ access resin.io's production downloads S3 bucket.
 Run the following command to publish a specific file:
 
 ```sh
-./scripts/publish/aws-s3.sh -f <file> -b <bucket> -v <version> -t <production|snapshot>
+./scripts/publish/aws-s3.sh -f <file> -b <bucket> -v <version> -p <product name>
 ```
 
 Or run the following command to publish all files for the current combination

docs/RUNNING-LOCALLY.md

@@ -33,7 +33,7 @@ The following MinGW packages are required:
 - `msys-make`
 - `msys-unzip`
 - `msys-zip`
-- `msys-wget`
+- `msys-curl`
 - `msys-bash`
 - `msys-coreutils`
 

scripts/build/docker/Dockerfile-i686

@@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y \
   python-pip \
   python-dev \
   unzip \
-  wget \
+  curl \
   xvfb \
   zip
 
@@ -28,4 +28,4 @@ RUN npm config set spin=false
 RUN npm install -g bower asar electron-installer-debian
 
 # Python
-RUN pip install codespell==1.9.2
+RUN pip install codespell==1.9.2 awscli

scripts/build/docker/Dockerfile-x86_64

@@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y \
   python-pip \
   python-dev \
   unzip \
-  wget \
+  curl \
   xvfb \
   zip
 
@@ -28,4 +28,4 @@ RUN npm config set spin=false
 RUN npm install -g bower asar electron-installer-debian
 
 # Python
-RUN pip install codespell==1.9.2
+RUN pip install codespell==1.9.2 awscli

scripts/build/docker/Dockerfile.template

@@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y \
   python-pip \
   python-dev \
   unzip \
-  wget \
+  curl \
   xvfb \
   zip
 
@@ -28,4 +28,4 @@ RUN npm config set spin=false
 RUN npm install -g bower asar electron-installer-debian
 
 # Python
-RUN pip install codespell==1.9.2
+RUN pip install codespell==1.9.2 awscli

scripts/build/docker/run-command.sh

@@ -82,6 +82,10 @@ fi
 # The `-t` and TERM setup is needed to display coloured output.
 docker run -t \
   --env "TERM=xterm-256color" \
+  --env "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" \
+  --env "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" \
+  --env "RELEASE_TYPE=$RELEASE_TYPE" \
+  --env "CI=$CI" \
   --cap-add SYS_ADMIN \
   --device /dev/fuse:/dev/fuse:mrw \
   --volume "$ARGV_SOURCE_CODE_DIRECTORY:/etcher" \

scripts/build/download-tool.sh

@@ -19,7 +19,7 @@
 set -u
 set -e
 
-./scripts/build/check-dependency.sh wget
+./scripts/build/check-dependency.sh curl
 
 SHA256SUM=$(./scripts/build/check-dependency.sh sha256sum "shasum -a 256")
 
@@ -78,7 +78,8 @@ if [ -f "$ARGV_OUTPUT" ]; then
   fi
 fi
 
-wget --no-check-certificate "$ARGV_URL" -O "$TEMP_OUTPUT"
+echo "Downloading $ARGV_URL"
+curl --continue-at - --retry 100 --location --output "$TEMP_OUTPUT" "$ARGV_URL"
 
 if ! checksum_matches "$TEMP_OUTPUT" "$ARGV_CHECKSUM"; then
   echo "Checksum mismatch" 1>&2

scripts/build/electron-download-package.sh

@@ -60,7 +60,7 @@ ELECTRON_ARCHITECTURE=$(./scripts/build/architecture-convert.sh -r "$ARGV_ARCHIT
 ELECTRON_GITHUB_REPOSITORY=https://github.com/electron/electron
 ELECTRON_DOWNLOADS_BASEURL="$ELECTRON_GITHUB_REPOSITORY/releases/download/v$ARGV_ELECTRON_VERSION"
 ELECTRON_FILENAME="electron-v$ARGV_ELECTRON_VERSION-$ARGV_OPERATING_SYSTEM-$ELECTRON_ARCHITECTURE.zip"
-ELECTRON_CHECKSUM=$(wget --no-check-certificate -O - "$ELECTRON_DOWNLOADS_BASEURL/SHASUMS256.txt" | grep "$ELECTRON_FILENAME" | cut -d ' ' -f 1)
+ELECTRON_CHECKSUM=$(curl --location "$ELECTRON_DOWNLOADS_BASEURL/SHASUMS256.txt" | grep "$ELECTRON_FILENAME" | cut -d ' ' -f 1)
 
 ./scripts/build/download-tool.sh \
   -u "$ELECTRON_DOWNLOADS_BASEURL/$ELECTRON_FILENAME" \

scripts/build/electron-installer-nsis-win32.sh

@@ -18,7 +18,6 @@
 
 set -u
 set -e
-set -x
 
 ./scripts/build/check-dependency.sh zip
 

scripts/publish/aws-s3.sh

@@ -31,6 +31,7 @@ function usage() {
   echo "    -b <s3 bucket>"
   echo "    -v <version>"
   echo "    -p <product name>"
+  echo "    -k [S3 key prefix]"
   exit 1
 }
 
@@ -38,13 +39,15 @@ ARGV_FILE=""
 ARGV_BUCKET=""
 ARGV_VERSION=""
 ARGV_PRODUCT_NAME=""
+ARGV_PREFIX=""
 
-while getopts ":f:b:v:p:" option; do
+while getopts ":f:b:v:p:k:" option; do
   case $option in
     f) ARGV_FILE="$OPTARG" ;;
     b) ARGV_BUCKET="$OPTARG" ;;
     v) ARGV_VERSION="$OPTARG" ;;
     p) ARGV_PRODUCT_NAME="$OPTARG" ;;
+    k) ARGV_PREFIX="$OPTARG" ;;
     *) usage ;;
   esac
 done
@@ -59,8 +62,19 @@ fi
 
 FILENAME=$(basename "$ARGV_FILE")
 
+if [ -n "$ARGV_PREFIX" ]; then
+  S3_KEY="$ARGV_PRODUCT_NAME/$ARGV_PREFIX/$ARGV_VERSION/$FILENAME"
+else
+  S3_KEY="$ARGV_PRODUCT_NAME/$ARGV_VERSION/$FILENAME"
+fi
+
 aws s3api put-object \
   --bucket "$ARGV_BUCKET" \
   --acl public-read \
-  --key "$ARGV_PRODUCT_NAME/$ARGV_VERSION/$FILENAME" \
+  --key "$S3_KEY" \
   --body "$ARGV_FILE"
+
+# Escape plus signs when printing the final URL
+URL="$(echo "https://$ARGV_BUCKET.s3.amazonaws.com/$S3_KEY" | sed 's/\+/%2B/g')"
+
+echo "Uploaded $(basename "$ARGV_FILE") to $URL"