Don't upload multiple times to same artifact in build workflow

The build workflow produces binaries for a range of target hosts. This is done by using a job matrix in the GitHub
Actions workflow that produces each build in a parallel job. GitHub Actions workflow artifacts are used to transfer the
generated files between sequential jobs in the workflow. The "actions/upload-artifact" action is used for this purpose.

Previously, a single artifact was used for this purpose, with each of the parallel jobs uploading its own generated
files to that artifact. However, support for uploading multiple times to a single artifact was dropped in version 4.0.0
of the "actions/upload-artifact" action. So it is now necessary to use a dedicated artifact for each of the builds.
These can be downloaded in aggregate by using the artifact name globbing and merging features which were introduced in
version 4.1.0 of the "actions/download-artifact" action.
This commit is contained in:
per1234 2024-11-21 00:04:20 -08:00 committed by Per Tillisch
parent 90d3d77ca4
commit dba57b312c

View File

@ -48,9 +48,9 @@ env:
GO_VERSION: '1.21' GO_VERSION: '1.21'
# See: https://github.com/actions/setup-node/#readme # See: https://github.com/actions/setup-node/#readme
NODE_VERSION: '18.17' NODE_VERSION: '18.17'
JOB_TRANSFER_ARTIFACT: build-artifacts JOB_TRANSFER_ARTIFACT_PREFIX: build-artifacts-
CHANGELOG_ARTIFACTS: changelog CHANGELOG_ARTIFACTS: changelog
STAGED_CHANNEL_FILES_ARTIFACT: staged-channel-files STAGED_CHANNEL_FILE_ARTIFACT_PREFIX: staged-channel-file-
BASE_BUILD_DATA: | BASE_BUILD_DATA: |
- config: - config:
# Human identifier for the job. # Human identifier for the job.
@ -68,6 +68,8 @@ env:
certificate-extension: pfx certificate-extension: pfx
# Container for windows cert signing # Container for windows cert signing
certificate-container: INSTALLER_CERT_WINDOWS_CONTAINER certificate-container: INSTALLER_CERT_WINDOWS_CONTAINER
# Arbitrary identifier used to give the workflow artifact uploaded by each "build" matrix job a unique name.
job-transfer-artifact-suffix: Windows_64bit
# Quoting on the value is required here to allow the same comparison expression syntax to be used for this # Quoting on the value is required here to allow the same comparison expression syntax to be used for this
# and the companion needs.select-targets.outputs.merge-channel-files property (output values always have string # and the companion needs.select-targets.outputs.merge-channel-files property (output values always have string
# type). # type).
@ -91,6 +93,7 @@ env:
{ {
\"image\": \"ghcr.io/arduino/arduino-ide/linux:main\" \"image\": \"ghcr.io/arduino/arduino-ide/linux:main\"
} }
job-transfer-artifact-suffix: Linux_64bit
mergeable-channel-file: 'false' mergeable-channel-file: 'false'
artifacts: artifacts:
- path: '*Linux_64bit.zip' - path: '*Linux_64bit.zip'
@ -107,6 +110,7 @@ env:
certificate-secret: APPLE_SIGNING_CERTIFICATE_P12 certificate-secret: APPLE_SIGNING_CERTIFICATE_P12
certificate-password-secret: KEYCHAIN_PASSWORD certificate-password-secret: KEYCHAIN_PASSWORD
certificate-extension: p12 certificate-extension: p12
job-transfer-artifact-suffix: macOS_64bit
mergeable-channel-file: 'true' mergeable-channel-file: 'true'
artifacts: artifacts:
- path: '*macOS_64bit.dmg' - path: '*macOS_64bit.dmg'
@ -121,6 +125,7 @@ env:
certificate-secret: APPLE_SIGNING_CERTIFICATE_P12 certificate-secret: APPLE_SIGNING_CERTIFICATE_P12
certificate-password-secret: KEYCHAIN_PASSWORD certificate-password-secret: KEYCHAIN_PASSWORD
certificate-extension: p12 certificate-extension: p12
job-transfer-artifact-suffix: macOS_arm64
mergeable-channel-file: 'true' mergeable-channel-file: 'true'
artifacts: artifacts:
- path: '*macOS_arm64.dmg' - path: '*macOS_arm64.dmg'
@ -233,7 +238,7 @@ jobs:
) | \ ) | \
yq \ yq \
--output-format json \ --output-format json \
'[.[].artifacts.[]]' 'map(.artifacts[] + (.config | pick(["job-transfer-artifact-suffix"])))'
)" )"
# The build matrix produces two macOS jobs (x86 and ARM) so the "channel update info files" # The build matrix produces two macOS jobs (x86 and ARM) so the "channel update info files"
@ -252,7 +257,7 @@ jobs:
echo "${{ env.BASE_BUILD_DATA }}" | \ echo "${{ env.BASE_BUILD_DATA }}" | \
yq \ yq \
--output-format json \ --output-format json \
'[.[].artifacts.[]]' 'map(.artifacts[] + (.config | pick(["job-transfer-artifact-suffix"])))'
)" )"
merge_channel_files="false" merge_channel_files="false"
@ -417,13 +422,13 @@ jobs:
matrix.config.mergeable-channel-file == 'true' matrix.config.mergeable-channel-file == 'true'
with: with:
if-no-files-found: error if-no-files-found: error
name: ${{ env.STAGED_CHANNEL_FILES_ARTIFACT }} name: ${{ env.STAGED_CHANNEL_FILE_ARTIFACT_PREFIX }}${{ matrix.config.job-transfer-artifact-suffix }}
path: ${{ matrix.config.working-directory && format('{0}/{1}', matrix.config.working-directory, env.STAGED_CHANNEL_FILES_PATH) || env.STAGED_CHANNEL_FILES_PATH }} path: ${{ matrix.config.working-directory && format('{0}/{1}', matrix.config.working-directory, env.STAGED_CHANNEL_FILES_PATH) || env.STAGED_CHANNEL_FILES_PATH }}
- name: Upload [GitHub Actions] - name: Upload builds to job transfer artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ env.JOB_TRANSFER_ARTIFACT }} name: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}${{ matrix.config.job-transfer-artifact-suffix }}
path: ${{ matrix.config.working-directory && format('{0}/{1}', matrix.config.working-directory, env.BUILD_ARTIFACTS_PATH) || env.BUILD_ARTIFACTS_PATH }} path: ${{ matrix.config.working-directory && format('{0}/{1}', matrix.config.working-directory, env.BUILD_ARTIFACTS_PATH) || env.BUILD_ARTIFACTS_PATH }}
- name: Manual Clean up for self-hosted runners - name: Manual Clean up for self-hosted runners
@ -449,16 +454,17 @@ jobs:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Download staged-for-merge channel files artifact - name: Download staged-for-merge channel file artifacts
uses: actions/download-artifact@v4 uses: actions/download-artifact@v4
with: with:
name: ${{ env.STAGED_CHANNEL_FILES_ARTIFACT }} merge-multiple: true
path: ${{ env.CHANNEL_FILES_PATH }} path: ${{ env.CHANNEL_FILES_PATH }}
pattern: ${{ env.STAGED_CHANNEL_FILE_ARTIFACT_PREFIX }}*
- name: Remove no longer needed artifact - name: Remove no longer needed artifacts
uses: geekyeggo/delete-artifact@v5 uses: geekyeggo/delete-artifact@v5
with: with:
name: ${{ env.STAGED_CHANNEL_FILES_ARTIFACT }} name: ${{ env.STAGED_CHANNEL_FILE_ARTIFACT_PREFIX }}*
- name: Install Node.js - name: Install Node.js
uses: actions/setup-node@v4 uses: actions/setup-node@v4
@ -488,11 +494,11 @@ jobs:
--channel "${{ needs.build-type-determination.outputs.channel-name }}" \ --channel "${{ needs.build-type-determination.outputs.channel-name }}" \
--input "${{ env.CHANNEL_FILES_PATH }}" --input "${{ env.CHANNEL_FILES_PATH }}"
- name: Upload merged channel files to job transfer artifact - name: Upload merged channel files job transfer artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
if-no-files-found: error if-no-files-found: error
name: ${{ env.JOB_TRANSFER_ARTIFACT }} name: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}channel-files
path: ${{ env.CHANNEL_FILES_PATH }} path: ${{ env.CHANNEL_FILES_PATH }}
artifacts: artifacts:
@ -503,22 +509,25 @@ jobs:
if: always() && needs.build.result != 'skipped' if: always() && needs.build.result != 'skipped'
runs-on: ubuntu-latest runs-on: ubuntu-latest
env:
BUILD_ARTIFACTS_FOLDER: build-artifacts
strategy: strategy:
matrix: matrix:
artifact: ${{ fromJson(needs.select-targets.outputs.artifact-matrix) }} artifact: ${{ fromJson(needs.select-targets.outputs.artifact-matrix) }}
steps: steps:
- name: Download job transfer artifact - name: Download job transfer artifact that contains ${{ matrix.artifact.name }} tester build
uses: actions/download-artifact@v4 uses: actions/download-artifact@v4
with: with:
name: ${{ env.JOB_TRANSFER_ARTIFACT }} name: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}${{ matrix.artifact.job-transfer-artifact-suffix }}
path: ${{ env.JOB_TRANSFER_ARTIFACT }} path: ${{ env.BUILD_ARTIFACTS_FOLDER }}
- name: Upload tester build artifact - name: Upload tester build artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.artifact.name }} name: ${{ matrix.artifact.name }}
path: ${{ env.JOB_TRANSFER_ARTIFACT }}/${{ matrix.artifact.path }} path: ${{ env.BUILD_ARTIFACTS_FOLDER }}/${{ matrix.artifact.path }}
changelog: changelog:
needs: needs:
@ -561,11 +570,11 @@ jobs:
echo "$BODY" > CHANGELOG.txt echo "$BODY" > CHANGELOG.txt
- name: Upload Changelog [GitHub Actions] - name: Upload changelog job transfer artifact
if: needs.build-type-determination.outputs.is-nightly == 'true' if: needs.build-type-determination.outputs.is-nightly == 'true'
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ env.JOB_TRANSFER_ARTIFACT }} name: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}changelog
path: CHANGELOG.txt path: CHANGELOG.txt
publish: publish:
@ -584,18 +593,23 @@ jobs:
needs.build-type-determination.outputs.publish-to-s3 == 'true' && needs.build-type-determination.outputs.publish-to-s3 == 'true' &&
needs.build-type-determination.outputs.is-nightly == 'true' needs.build-type-determination.outputs.is-nightly == 'true'
runs-on: ubuntu-latest runs-on: ubuntu-latest
env:
ARTIFACTS_FOLDER: build-artifacts
steps: steps:
- name: Download [GitHub Actions] - name: Download all job transfer artifacts
uses: actions/download-artifact@v4 uses: actions/download-artifact@v4
with: with:
name: ${{ env.JOB_TRANSFER_ARTIFACT }} merge-multiple: true
path: ${{ env.JOB_TRANSFER_ARTIFACT }} path: ${{ env.ARTIFACTS_FOLDER }}
pattern: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}*
- name: Publish Nightly [S3] - name: Publish Nightly [S3]
uses: docker://plugins/s3 uses: docker://plugins/s3
env: env:
PLUGIN_SOURCE: '${{ env.JOB_TRANSFER_ARTIFACT }}/*' PLUGIN_SOURCE: '${{ env.ARTIFACTS_FOLDER }}/*'
PLUGIN_STRIP_PREFIX: '${{ env.JOB_TRANSFER_ARTIFACT }}/' PLUGIN_STRIP_PREFIX: '${{ env.ARTIFACTS_FOLDER }}/'
PLUGIN_TARGET: '/arduino-ide/nightly' PLUGIN_TARGET: '/arduino-ide/nightly'
PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }} PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@ -616,12 +630,17 @@ jobs:
needs.changelog.result == 'success' && needs.changelog.result == 'success' &&
needs.build-type-determination.outputs.is-release == 'true' needs.build-type-determination.outputs.is-release == 'true'
runs-on: ubuntu-latest runs-on: ubuntu-latest
env:
ARTIFACTS_FOLDER: build-artifacts
steps: steps:
- name: Download [GitHub Actions] - name: Download all job transfer artifacts
uses: actions/download-artifact@v4 uses: actions/download-artifact@v4
with: with:
name: ${{ env.JOB_TRANSFER_ARTIFACT }} merge-multiple: true
path: ${{ env.JOB_TRANSFER_ARTIFACT }} path: ${{ env.ARTIFACTS_FOLDER }}
pattern: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}*
- name: Get Tag - name: Get Tag
id: tag_name id: tag_name
@ -633,7 +652,7 @@ jobs:
with: with:
repo_token: ${{ secrets.GITHUB_TOKEN }} repo_token: ${{ secrets.GITHUB_TOKEN }}
release_name: ${{ steps.tag_name.outputs.TAG_NAME }} release_name: ${{ steps.tag_name.outputs.TAG_NAME }}
file: ${{ env.JOB_TRANSFER_ARTIFACT }}/* file: ${{ env.ARTIFACTS_FOLDER }}/*
tag: ${{ github.ref }} tag: ${{ github.ref }}
file_glob: true file_glob: true
body: ${{ needs.changelog.outputs.BODY }} body: ${{ needs.changelog.outputs.BODY }}
@ -642,8 +661,8 @@ jobs:
if: needs.build-type-determination.outputs.publish-to-s3 == 'true' if: needs.build-type-determination.outputs.publish-to-s3 == 'true'
uses: docker://plugins/s3 uses: docker://plugins/s3
env: env:
PLUGIN_SOURCE: '${{ env.JOB_TRANSFER_ARTIFACT }}/*' PLUGIN_SOURCE: '${{ env.ARTIFACTS_FOLDER }}/*'
PLUGIN_STRIP_PREFIX: '${{ env.JOB_TRANSFER_ARTIFACT }}/' PLUGIN_STRIP_PREFIX: '${{ env.ARTIFACTS_FOLDER }}/'
PLUGIN_TARGET: '/arduino-ide' PLUGIN_TARGET: '/arduino-ide'
PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }} PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@ -661,7 +680,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Remove unneeded job transfer artifact - name: Remove unneeded job transfer artifacts
uses: geekyeggo/delete-artifact@v5 uses: geekyeggo/delete-artifact@v5
with: with:
name: ${{ env.JOB_TRANSFER_ARTIFACT }} name: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}*