fwupdate: use published flag instead of latest

board/common/overlay/sbin/fwupdate

@@ -211,7 +211,7 @@ function show_versions() {
     show_json=$1
 
     # the /usr/libexec/list-versions-* helpers return a table with the following format:
-    #     <version>|<prerelease>|<board>|<url>|<date|latest>
+    #     <version>|<prerelease>|<board>|<url>|<date|published>
     versions=$(FW_USERNAME=${OS_FIRMWARE_USERNAME} FW_PASSWORD=${OS_FIRMWARE_PASSWORD} \
                /usr/libexec/list-versions-${OS_FIRMWARE_METHOD} ${OS_FIRMWARE_REPO})
 
@@ -223,6 +223,9 @@ function show_versions() {
         if [[ "${OS_PRERELEASES}" == "false" && "${varr[1]}" == "true" ]]; then
             continue  # skip prereleases
         fi
+        if [[ "${varr[5]}" == "false" ]]; then
+            continue  # skip unpublished
+        fi
         if [[ "${board}" != "${varr[2]}" ]]; then
             continue  # skip other boards
         fi
@@ -231,8 +234,7 @@ function show_versions() {
             echo "{\"version\": \"${varr[0]}\"," \
                   "\"url\": \"${varr[3]}\"," \
                   "\"prerelease\": ${varr[1]:-false}," \
-                  "\"date\": \"${varr[4]}\"," \
+                  "\"date\": \"${varr[4]}\"}"
-                  "\"latest\": ${varr[5]:-false}}"
         else
             echo ${varr[0]}
         fi

board/common/overlay/usr/libexec/list-versions-bitbucket

@@ -14,24 +14,9 @@ function check_prerelease() {
     fi
 }
 
-function add_latest_flag() {
+function add_published_flag() {
-    declare -A latest_stable_by_board
+    read line
-    declare -A latest_prerelease_by_board
+    echo "${line}|true"
-    while read line; do
-        _IFS=${IFS} IFS="|" line_arr=(${line}) IFS=${_IFS}
-        latest="false"
-        prerelease=${line_arr[1]}
-        board=${line_arr[2]}
-        if [[ "${prerelease}" == "true" && -z "${latest_prerelease_by_board[${board}]}" ]]; then
-            latest="true"
-            latest_prerelease_by_board[${board}]="true"
-        elif [[ "${prerelease}" == "false" && -z "${latest_stable_by_board[${board}]}" ]]; then
-            latest="true"
-            latest_stable_by_board[${board}]="true"
-        fi
-        
-        echo "${line}|${latest}"
-    done
 }
 
 extensions=".img.gz .img.xz .img"
@@ -43,5 +28,5 @@ rtrimstr=$(for e in ${extensions}; do echo -n " | rtrimstr(\"${e}\")"; done)
 jq_expr=".values[] | [{a: .name | split(\"-\"), url: .links.self.href, date: .created_on | split(\"T\")[0]}] |
          map((.a[2] ${rtrimstr}), \"false\", .a[1], .url, .date) | join(\"|\")"
 
-curl ${opts} ${url} | jq --raw-output "${jq_expr}" | while read line; do echo "${line}" | check_prerelease | add_latest_flag; done
+curl ${opts} ${url} | jq --raw-output "${jq_expr}" | while read line; do echo "${line}" | check_prerelease | add_published_flag; done
 exit ${PIPESTATUS[0]}

board/common/overlay/usr/libexec/list-versions-github

@@ -8,21 +8,16 @@ fi
 opts="-s -S -f"
 test -n "${FW_USERNAME}" && opts+=" --user ${FW_USERNAME}:${FW_PASSWORD}"
 releases_url=https://api.github.com/repos/$1/releases
-latest_url=https://api.github.com/repos/$1/releases/latest
 
 # Get release info via GitHub API
-latest_version=$(curl ${opts} ${latest_url} | jq --raw-output ".name")
 releases=$(curl ${opts} ${releases_url})
 test $? == 0 || exit 1
 
-# Latest pre-release is the first one
-latest_prerelease=$(jq --raw-output | '[.[] | select(.prerelease)] | first | .name' <<< ${releases})
-
 # Format release lines
 jq_expr='.[] | {version: .name, prerelease: .prerelease | tostring} + 
          (.assets[] | {name: .name | split("-")[1], url: .browser_download_url}) + 
          ({date: .created_at | split("T")[0]}) +
-         ({latest: (.name=="'${latest_version}'")})
+         ({published: true})
          | flatten | join("|")'
 
 jq --raw-output "${jq_expr}" <<< ${releases}

board/common/overlay/usr/libexec/list-versions-s3

@@ -13,34 +13,17 @@ bucket=$(echo ${bucket_path} | cut -d '/' -f 1)
 path=${bucket_path:${#bucket} + 1}
 
 opts="-s -S -f"
+max_keys="1000"
 test -n "${FW_USERNAME}" && opts+=" --user ${FW_USERNAME}:${FW_PASSWORD}"
 url="https://${bucket}.s3.amazonaws.com"
 
-xml_result=$(curl ${opts} "${url}?list-type=2&prefix=${path}")
+xml_result=$(curl ${opts} "${url}?list-type=2&max_keys=${max_keys}&prefix=${path}&fetch-owner=true")
 keys=$(echo "${xml_result}" | grep -oE '<Key>[^<]+<\/Key>' | sed 's/<Key>\(.*\)<\/Key>/\1/')
 dates=$(echo "${xml_result}" | grep -oE '<LastModified>[^<]+<\/LastModified>' | sed 's/<LastModified>\(.*\)<\/LastModified>/\1/')
+storage_classes=$(echo "${xml_result}" | grep -oE '<StorageClass>[^<]+<\/StorageClass>' | sed 's/<StorageClass>\(.*\)<\/StorageClass>/\1/')
 dates=(${dates})
 files=(${keys})
-
+storage_classes=(${storage_classes})
-function add_latest_flag() {
-    declare -A latest_stable_by_board
-    declare -A latest_prerelease_by_board
-    while read line; do
-        _IFS=${IFS} IFS="|" line_arr=(${line}) IFS=${_IFS}
-        latest="false"
-        prerelease=${line_arr[1]}
-        board=${line_arr[2]}
-        if [[ "${prerelease}" == "true" && -z "${latest_prerelease_by_board[${board}]}" ]]; then
-            latest="true"
-            latest_prerelease_by_board[${board}]="true"
-        elif [[ "${prerelease}" == "false" && -z "${latest_stable_by_board[${board}]}" ]]; then
-            latest="true"
-            latest_stable_by_board[${board}]="true"
-        fi
-        
-        echo "${line}|${latest}"
-    done
-}
 
 i=0
 for file in ${files[*]}; do
@@ -54,12 +37,19 @@ for file in ${files[*]}; do
        [[ "${version}" =~ ^(dev|nightly).*$ ]]; then # e.g. dev20180314, nightly20180314
         prerelease=true
     fi
+    
+    # Items with REDUCED_REDUNDANCY storage class are considered, by convention, unpublished
+    published="true"
+    storage_class=${storage_classes[${i}]}
+    if [[ "${storage_class}" == "REDUCED_REDUNDANCY" ]]; then
+        published="false"
+    fi
 
     [[ "${fname}" =~ ^([^-]+)-([^-]+)-(.+)$ ]] || continue
     board="${BASH_REMATCH[2]}"
     final_url="${url}/${path}/${version}/${fname}"
     date="${dates[${i}]:0:10}"
 
-    echo "${version}|${prerelease}|${board}|${final_url}|${date}"
+    echo "${version}|${prerelease}|${board}|${final_url}|${date}|${published}"
     i=$((i + 1))
-done | semver-sort -r -t '|' -k 1 | add_latest_flag
+done | semver-sort -r -t '|' -k 1