jeans/etcher
jeans/etcher
1
0
Fork 0
mirror of https://github.com/balena-io/etcher.git synced 2025-07-23 19:26:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: publish snapshot builds to S3 (#1078)

Browse Source 
This commit makes Appveyor and Travis CI publish snapshot builds to S3
when a pull request is merged, by making use of the `publish-aws-s3`
Makefile target.

The changes required for such type of deployment are the followings:

- Set `S3_BUCKET` to `resin-nightly-downloads` when doing snapshot
  builds

- Add deploy sections to `.travis.yml` and `appveyor.yml` that run `make
  publish-aws-s3`

- Don't change `PRODUCT_NAME` when doing snapshot builds (given we'll be
  publishing to a different S3 bucket)

- Install `awscli` in Appveyor CI and Travis CI

- Make GNU/Linux Docker containers inherit `AWS_ACCESS_KEY_ID` and
  `AWS_SECRET_ACCESS_KEY` from the environment (so `awscli` is
  configured correctly inside them)

- Add a prefix option to `aws-s3.sh` publish script to prepend a string
  to the S3 path, so we can add a timestamp to more easily distinguish
  files inside the `resin-nightly-downloads` bucket

- Print the published URL from `aws-s3.sh` for convenience purposes, so
  we can click it when skimming through CI builds logs

- Add the `-R` and `-L` options when recursively copying `node_modules`
  during a snapshot build to prevent weird Appveyor errors related to
  hard links. The options listed before make sure that we recursively
  resolve every link while copying

- Move from `wget` to `curl` to avoid certificate check failures

Signed-off-by: Juan Cruz Viotti <jviotti@openmailbox.org>
This commit is contained in:
Juan Cruz Viotti 2017-03-21 18:35:44 -04:00 committed by GitHub
parent 5a01f4854c
commit 799ebc6aa0
13 changed files with 68 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
.travis.yml
View File

@ -33,28 +33,33 @@ before_install:
fi
install:
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
pip install codespell==1.9.2;
npm install -g bower;
brew install afsctool;
brew install jq;
fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
./scripts/build/docker/run-command.sh -r ${TARGET_ARCH} -s ${PWD} -c "make info && make electron-develop";
else
pip install codespell==1.9.2 awscli;
npm install -g bower asar;
brew install afsctool jq;
make info;
make electron-develop;
fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
./scripts/build/docker/run-command.sh -r ${TARGET_ARCH} -s ${PWD} -c "make info && make electron-develop";
fi
script:
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
make sanity-checks && npm test;
fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
./scripts/build/docker/run-command.sh -r ${TARGET_ARCH} -s ${PWD} -c "make sanity-checks && xvfb-run --server-args=$XVFB_ARGS npm test";
else
make sanity-checks && npm test;
fi
deploy:
provider: script
script: if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
./scripts/build/docker/run-command.sh -r ${TARGET_ARCH} -s ${PWD} -c "make publish-aws-s3";
else
make publish-aws-s3;
fi
on:
branch: master
notifications:
email: false
webhooks:

23
Makefile
View File

@ -26,17 +26,18 @@ APPLICATION_COPYRIGHT = $(shell jq -r '.copyright' package.json)
APPLICATION_CATEGORY = public.app-category.developer-tools
APPLICATION_BUNDLE_ID = io.resin.etcher
APPLICATION_FILES = lib,assets
S3_BUCKET = resin-production-downloads
# Add the current commit to the version if release type is "snapshot"
RELEASE_TYPE ?= snapshot
PACKAGE_JSON_VERSION = $(shell jq -r '.version' package.json)
ifeq ($(RELEASE_TYPE),production)
APPLICATION_VERSION = $(PACKAGE_JSON_VERSION)
S3_BUCKET = resin-production-downloads
endif
ifeq ($(RELEASE_TYPE),snapshot)
CURRENT_COMMIT_HASH = $(shell git log -1 --format="%h")
APPLICATION_VERSION = $(PACKAGE_JSON_VERSION)+$(CURRENT_COMMIT_HASH)
S3_BUCKET = resin-nightly-downloads
endif
ifndef APPLICATION_VERSION
$(error Invalid release type: $(RELEASE_TYPE))
@ -139,13 +140,7 @@ endif
TARGET_ARCH_DEBIAN = $(shell ./scripts/build/architecture-convert.sh -r $(TARGET_ARCH) -t debian)
ifeq ($(RELEASE_TYPE),production)
PRODUCT_NAME = etcher
endif
ifeq ($(RELEASE_TYPE),snapshot)
PRODUCT_NAME = etcher-snapshots
endif
PRODUCT_NAME = etcher
APPLICATION_NAME_LOWERCASE = $(shell echo $(APPLICATION_NAME) | tr A-Z a-z)
APPLICATION_VERSION_DEBIAN = $(shell echo $(APPLICATION_VERSION) | tr "-" "~")
@ -192,7 +187,7 @@ $(BUILD_DIRECTORY)/electron-$(TARGET_PLATFORM)-$(APPLICATION_VERSION)-$(TARGET_A
./scripts/build/electron-create-resources-app.sh -s . -o $@ \
-v $(APPLICATION_VERSION) \
-f "$(APPLICATION_FILES)"
$(foreach prerequisite,$^,$(call execute-command,cp -rf $(prerequisite) $@))
$(foreach prerequisite,$^,$(call execute-command,cp -RLf $(prerequisite) $@))
$(BUILD_DIRECTORY)/electron-$(TARGET_PLATFORM)-$(APPLICATION_VERSION)-$(TARGET_ARCH)-app.asar: \
$(BUILD_DIRECTORY)/electron-$(TARGET_PLATFORM)-$(APPLICATION_VERSION)-$(TARGET_ARCH)-app \
@ -373,11 +368,21 @@ endif
ifdef PUBLISH_AWS_S3
publish-aws-s3: $(PUBLISH_AWS_S3)
ifeq ($(RELEASE_TYPE),production)
$(foreach publishable,$^,$(call execute-command,./scripts/publish/aws-s3.sh \
-f $(publishable) \
-b $(S3_BUCKET) \
-v $(APPLICATION_VERSION) \
-p $(PRODUCT_NAME)))
endif
ifeq ($(RELEASE_TYPE),snapshot)
$(foreach publishable,$^,$(call execute-command,./scripts/publish/aws-s3.sh \
-f $(publishable) \
-b $(S3_BUCKET) \
-v $(APPLICATION_VERSION) \
-p $(PRODUCT_NAME) \
-k $(shell date +"%Y-%m-%d")))
endif
TARGETS += publish-aws-s3
endif

5
appveyor.yml
View File

@ -28,7 +28,7 @@ install:
- set PATH=C:\Program Files (x86)\NSIS;%PATH%
- set PATH=C:\MinGW\bin;%PATH%
- set PATH=C:\MinGW\msys\1.0\bin;%PATH%
- pip install codespell==1.9.2
- pip install codespell==1.9.2 awscli
- make info
- make electron-develop
@ -40,6 +40,9 @@ test_script:
- make sanity-checks
- cmd: npm test
deploy_script:
- cmd: IF %APPVEYOR_REPO_BRANCH%==master (make publish-aws-s3)
notifications:
- provider: Webhook

2
docs/PUBLISHING.md
View File

@ -93,7 +93,7 @@ access resin.io's production downloads S3 bucket.
Run the following command to publish a specific file:
```sh
./scripts/publish/aws-s3.sh -f <file> -b <bucket> -v <version> -t <production|snapshot>
./scripts/publish/aws-s3.sh -f <file> -b <bucket> -v <version> -p <product name>
```
Or run the following command to publish all files for the current combination

2
docs/RUNNING-LOCALLY.md
View File

@ -33,7 +33,7 @@ The following MinGW packages are required:
- `msys-make`
- `msys-unzip`
- `msys-zip`
- `msys-wget`
- `msys-curl`
- `msys-bash`
- `msys-coreutils`

4
scripts/build/docker/Dockerfile-i686
View File

@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y \
python-pip \
python-dev \
unzip \
wget \
curl \
xvfb \
zip
@ -28,4 +28,4 @@ RUN npm config set spin=false
RUN npm install -g bower asar electron-installer-debian
# Python
RUN pip install codespell==1.9.2
RUN pip install codespell==1.9.2 awscli

4
scripts/build/docker/Dockerfile-x86_64
View File

@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y \
python-pip \
python-dev \
unzip \
wget \
curl \
xvfb \
zip
@ -28,4 +28,4 @@ RUN npm config set spin=false
RUN npm install -g bower asar electron-installer-debian
# Python
RUN pip install codespell==1.9.2
RUN pip install codespell==1.9.2 awscli

4
scripts/build/docker/Dockerfile.template
View File

@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y \
python-pip \
python-dev \
unzip \
wget \
curl \
xvfb \
zip
@ -28,4 +28,4 @@ RUN npm config set spin=false
RUN npm install -g bower asar electron-installer-debian
# Python
RUN pip install codespell==1.9.2
RUN pip install codespell==1.9.2 awscli

4
scripts/build/docker/run-command.sh
View File

@ -82,6 +82,10 @@ fi
# The `-t` and TERM setup is needed to display coloured output.
docker run -t \
--env "TERM=xterm-256color" \
--env "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" \
--env "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" \
--env "RELEASE_TYPE=$RELEASE_TYPE" \
--env "CI=$CI" \
--cap-add SYS_ADMIN \
--device /dev/fuse:/dev/fuse:mrw \
--volume "$ARGV_SOURCE_CODE_DIRECTORY:/etcher" \

5
scripts/build/download-tool.sh
View File

@ -19,7 +19,7 @@
set -u
set -e
./scripts/build/check-dependency.sh wget
./scripts/build/check-dependency.sh curl
SHA256SUM=$(./scripts/build/check-dependency.sh sha256sum "shasum -a 256")
@ -78,7 +78,8 @@ if [ -f "$ARGV_OUTPUT" ]; then
fi
fi
wget --no-check-certificate "$ARGV_URL" -O "$TEMP_OUTPUT"
echo "Downloading $ARGV_URL"
curl --continue-at - --retry 100 --location --output "$TEMP_OUTPUT" "$ARGV_URL"
if ! checksum_matches "$TEMP_OUTPUT" "$ARGV_CHECKSUM"; then
echo "Checksum mismatch" 1>&2

2
scripts/build/electron-download-package.sh
View File

@ -60,7 +60,7 @@ ELECTRON_ARCHITECTURE=$(./scripts/build/architecture-convert.sh -r "$ARGV_ARCHIT
ELECTRON_GITHUB_REPOSITORY=https://github.com/electron/electron
ELECTRON_DOWNLOADS_BASEURL="$ELECTRON_GITHUB_REPOSITORY/releases/download/v$ARGV_ELECTRON_VERSION"
ELECTRON_FILENAME="electron-v$ARGV_ELECTRON_VERSION-$ARGV_OPERATING_SYSTEM-$ELECTRON_ARCHITECTURE.zip"
ELECTRON_CHECKSUM=$(wget --no-check-certificate -O - "$ELECTRON_DOWNLOADS_BASEURL/SHASUMS256.txt" | grep "$ELECTRON_FILENAME" | cut -d ' ' -f 1)
ELECTRON_CHECKSUM=$(curl --location "$ELECTRON_DOWNLOADS_BASEURL/SHASUMS256.txt" | grep "$ELECTRON_FILENAME" | cut -d ' ' -f 1)
./scripts/build/download-tool.sh \
-u "$ELECTRON_DOWNLOADS_BASEURL/$ELECTRON_FILENAME" \

1
scripts/build/electron-installer-nsis-win32.sh
View File

@ -18,7 +18,6 @@
set -u
set -e
set -x
./scripts/build/check-dependency.sh zip

18
scripts/publish/aws-s3.sh
View File

@ -31,6 +31,7 @@ function usage() {
echo " -b <s3 bucket>"
echo " -v <version>"
echo " -p <product name>"
echo " -k [S3 key prefix]"
exit 1
}
@ -38,13 +39,15 @@ ARGV_FILE=""
ARGV_BUCKET=""
ARGV_VERSION=""
ARGV_PRODUCT_NAME=""
ARGV_PREFIX=""
while getopts ":f:b:v:p:" option; do
while getopts ":f:b:v:p:k:" option; do
case $option in
f) ARGV_FILE="$OPTARG" ;;
b) ARGV_BUCKET="$OPTARG" ;;
v) ARGV_VERSION="$OPTARG" ;;
p) ARGV_PRODUCT_NAME="$OPTARG" ;;
k) ARGV_PREFIX="$OPTARG" ;;
*) usage ;;
esac
done
@ -59,8 +62,19 @@ fi
FILENAME=$(basename "$ARGV_FILE")
if [ -n "$ARGV_PREFIX" ]; then
S3_KEY="$ARGV_PRODUCT_NAME/$ARGV_PREFIX/$ARGV_VERSION/$FILENAME"
else
S3_KEY="$ARGV_PRODUCT_NAME/$ARGV_VERSION/$FILENAME"
fi
aws s3api put-object \
--bucket "$ARGV_BUCKET" \
--acl public-read \
--key "$ARGV_PRODUCT_NAME/$ARGV_VERSION/$FILENAME" \
--key "$S3_KEY" \
--body "$ARGV_FILE"
# Escape plus signs when printing the final URL
URL="$(echo "https://$ARGV_BUCKET.s3.amazonaws.com/$S3_KEY" | sed 's/\+/%2B/g')"
echo "Uploaded $(basename "$ARGV_FILE") to $URL"