From 923b9316479cfa928a3980ebe3c1610bbdaa4e4a Mon Sep 17 00:00:00 2001 From: Calin Crisan Date: Sat, 11 Apr 2020 15:34:23 +0300 Subject: [PATCH] fwupdate: use published flag instead of latest --- board/common/overlay/sbin/fwupdate | 8 +++-- .../usr/libexec/list-versions-bitbucket | 23 +++--------- .../overlay/usr/libexec/list-versions-github | 7 +--- .../overlay/usr/libexec/list-versions-s3 | 36 +++++++------------ 4 files changed, 23 insertions(+), 51 deletions(-) diff --git a/board/common/overlay/sbin/fwupdate b/board/common/overlay/sbin/fwupdate index c41cfe1fa8..d74e5025c9 100755 --- a/board/common/overlay/sbin/fwupdate +++ b/board/common/overlay/sbin/fwupdate @@ -211,7 +211,7 @@ function show_versions() { show_json=$1 # the /usr/libexec/list-versions-* helpers return a table with the following format: - # |||| + # |||| versions=$(FW_USERNAME=${OS_FIRMWARE_USERNAME} FW_PASSWORD=${OS_FIRMWARE_PASSWORD} \ /usr/libexec/list-versions-${OS_FIRMWARE_METHOD} ${OS_FIRMWARE_REPO}) @@ -223,6 +223,9 @@ function show_versions() { if [[ "${OS_PRERELEASES}" == "false" && "${varr[1]}" == "true" ]]; then continue # skip prereleases fi + if [[ "${varr[5]}" == "false" ]]; then + continue # skip unpublished + fi if [[ "${board}" != "${varr[2]}" ]]; then continue # skip other boards fi @@ -231,8 +234,7 @@ function show_versions() { echo "{\"version\": \"${varr[0]}\"," \ "\"url\": \"${varr[3]}\"," \ "\"prerelease\": ${varr[1]:-false}," \ - "\"date\": \"${varr[4]}\"," \ - "\"latest\": ${varr[5]:-false}}" + "\"date\": \"${varr[4]}\"}" else echo ${varr[0]} fi diff --git a/board/common/overlay/usr/libexec/list-versions-bitbucket b/board/common/overlay/usr/libexec/list-versions-bitbucket index adf939912a..68615a8750 100755 --- a/board/common/overlay/usr/libexec/list-versions-bitbucket +++ b/board/common/overlay/usr/libexec/list-versions-bitbucket @@ -14,24 +14,9 @@ function check_prerelease() { fi } -function add_latest_flag() { - declare -A latest_stable_by_board - declare -A latest_prerelease_by_board - while read line; do - _IFS=${IFS} IFS="|" line_arr=(${line}) IFS=${_IFS} - latest="false" - prerelease=${line_arr[1]} - board=${line_arr[2]} - if [[ "${prerelease}" == "true" && -z "${latest_prerelease_by_board[${board}]}" ]]; then - latest="true" - latest_prerelease_by_board[${board}]="true" - elif [[ "${prerelease}" == "false" && -z "${latest_stable_by_board[${board}]}" ]]; then - latest="true" - latest_stable_by_board[${board}]="true" - fi - - echo "${line}|${latest}" - done +function add_published_flag() { + read line + echo "${line}|true" } extensions=".img.gz .img.xz .img" @@ -43,5 +28,5 @@ rtrimstr=$(for e in ${extensions}; do echo -n " | rtrimstr(\"${e}\")"; done) jq_expr=".values[] | [{a: .name | split(\"-\"), url: .links.self.href, date: .created_on | split(\"T\")[0]}] | map((.a[2] ${rtrimstr}), \"false\", .a[1], .url, .date) | join(\"|\")" -curl ${opts} ${url} | jq --raw-output "${jq_expr}" | while read line; do echo "${line}" | check_prerelease | add_latest_flag; done +curl ${opts} ${url} | jq --raw-output "${jq_expr}" | while read line; do echo "${line}" | check_prerelease | add_published_flag; done exit ${PIPESTATUS[0]} diff --git a/board/common/overlay/usr/libexec/list-versions-github b/board/common/overlay/usr/libexec/list-versions-github index 85267e25e1..58a183208e 100755 --- a/board/common/overlay/usr/libexec/list-versions-github +++ b/board/common/overlay/usr/libexec/list-versions-github @@ -8,21 +8,16 @@ fi opts="-s -S -f" test -n "${FW_USERNAME}" && opts+=" --user ${FW_USERNAME}:${FW_PASSWORD}" releases_url=https://api.github.com/repos/$1/releases -latest_url=https://api.github.com/repos/$1/releases/latest # Get release info via GitHub API -latest_version=$(curl ${opts} ${latest_url} | jq --raw-output ".name") releases=$(curl ${opts} ${releases_url}) test $? == 0 || exit 1 -# Latest pre-release is the first one -latest_prerelease=$(jq --raw-output | '[.[] | select(.prerelease)] | first | .name' <<< ${releases}) - # Format release lines jq_expr='.[] | {version: .name, prerelease: .prerelease | tostring} + (.assets[] | {name: .name | split("-")[1], url: .browser_download_url}) + ({date: .created_at | split("T")[0]}) + - ({latest: (.name=="'${latest_version}'")}) + ({published: true}) | flatten | join("|")' jq --raw-output "${jq_expr}" <<< ${releases} diff --git a/board/common/overlay/usr/libexec/list-versions-s3 b/board/common/overlay/usr/libexec/list-versions-s3 index 37eb905579..160168a313 100755 --- a/board/common/overlay/usr/libexec/list-versions-s3 +++ b/board/common/overlay/usr/libexec/list-versions-s3 @@ -13,34 +13,17 @@ bucket=$(echo ${bucket_path} | cut -d '/' -f 1) path=${bucket_path:${#bucket} + 1} opts="-s -S -f" +max_keys="1000" test -n "${FW_USERNAME}" && opts+=" --user ${FW_USERNAME}:${FW_PASSWORD}" url="https://${bucket}.s3.amazonaws.com" -xml_result=$(curl ${opts} "${url}?list-type=2&prefix=${path}") +xml_result=$(curl ${opts} "${url}?list-type=2&max_keys=${max_keys}&prefix=${path}&fetch-owner=true") keys=$(echo "${xml_result}" | grep -oE '[^<]+<\/Key>' | sed 's/\(.*\)<\/Key>/\1/') dates=$(echo "${xml_result}" | grep -oE '[^<]+<\/LastModified>' | sed 's/\(.*\)<\/LastModified>/\1/') +storage_classes=$(echo "${xml_result}" | grep -oE '[^<]+<\/StorageClass>' | sed 's/\(.*\)<\/StorageClass>/\1/') dates=(${dates}) files=(${keys}) - -function add_latest_flag() { - declare -A latest_stable_by_board - declare -A latest_prerelease_by_board - while read line; do - _IFS=${IFS} IFS="|" line_arr=(${line}) IFS=${_IFS} - latest="false" - prerelease=${line_arr[1]} - board=${line_arr[2]} - if [[ "${prerelease}" == "true" && -z "${latest_prerelease_by_board[${board}]}" ]]; then - latest="true" - latest_prerelease_by_board[${board}]="true" - elif [[ "${prerelease}" == "false" && -z "${latest_stable_by_board[${board}]}" ]]; then - latest="true" - latest_stable_by_board[${board}]="true" - fi - - echo "${line}|${latest}" - done -} +storage_classes=(${storage_classes}) i=0 for file in ${files[*]}; do @@ -54,12 +37,19 @@ for file in ${files[*]}; do [[ "${version}" =~ ^(dev|nightly).*$ ]]; then # e.g. dev20180314, nightly20180314 prerelease=true fi + + # Items with REDUCED_REDUNDANCY storage class are considered, by convention, unpublished + published="true" + storage_class=${storage_classes[${i}]} + if [[ "${storage_class}" == "REDUCED_REDUNDANCY" ]]; then + published="false" + fi [[ "${fname}" =~ ^([^-]+)-([^-]+)-(.+)$ ]] || continue board="${BASH_REMATCH[2]}" final_url="${url}/${path}/${version}/${fname}" date="${dates[${i}]:0:10}" - echo "${version}|${prerelease}|${board}|${final_url}|${date}" + echo "${version}|${prerelease}|${board}|${final_url}|${date}|${published}" i=$((i + 1)) -done | semver-sort -r -t '|' -k 1 | add_latest_flag +done | semver-sort -r -t '|' -k 1