[skip changelog] Update workflow and script to fetch Arduino CDN download data

.github/tools/fetch_athena_stats.sh

@@ -1,118 +0,0 @@
-#!/usr/bin/env bash
-
-# This script performs the following:
-# 1. Run the query, use jq to capture the QueryExecutionId, and then capture that into bash variable
-# 2. Wait for the query to finish running (240 seconds).
-# 3. Get the results.
-# 4. Json data points struct build
-
-# Expected env variables are:
-# AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for accessing AWS resources
-# AWS_ATHENA_SOURCE_TABLE
-# AWS_ATHENA_OUTPUT_LOCATION
-# GITHUB_REPOSITORY
-
-set -euo pipefail
-
-loadExecutionId=$(
-  aws athena start-query-execution \
-    --query-string "MSCK REPAIR TABLE ${AWS_ATHENA_SOURCE_TABLE};" \
-    --result-configuration "OutputLocation=${AWS_ATHENA_OUTPUT_LOCATION}" \
-    --region us-east-1 | jq -r ".QueryExecutionId"
-)
-
-echo "QueryExecutionId is ${loadExecutionId}"
-for i in $(seq 1 120); do
-  loadState=$(
-    aws athena get-query-execution \
-      --query-execution-id "${loadExecutionId}" \
-      --region us-east-1 | jq -r ".QueryExecution.Status.State"
-  )
-
-  if [[ "${loadState}" == "SUCCEEDED" ]]; then
-    break
-  fi
-
-  echo "QueryExecutionId ${loadExecutionId} - state is ${loadState}"
-
-  if [[ "${loadState}" == "FAILED" ]]; then
-    exit 1
-  fi
-
-  sleep 2
-done
-
-! read -r -d '' query <<EOM
-SELECT split_part(replace(json_extract_scalar(url_decode(url_decode(querystring)),'$.data.url'), 'https://downloads.arduino.cc/arduino-ide/arduino-ide_', ''),'?',1) AS flavor, count(json_extract(url_decode(url_decode(querystring)),'$')) AS gauge
-FROM stats_ingest_prod.complete_cf_logs_partitioned
-WHERE json_extract_scalar(url_decode(url_decode(querystring)),'$.data.url') LIKE 'https://downloads.arduino.cc/arduino-ide/arduino-ide_%'
-        AND json_extract_scalar(url_decode(url_decode(querystring)),'$.data.url') NOT LIKE '%latest%' -- exclude latest redirect
-group by 1 ;
-EOM
-
-queryExecutionId=$(
-  aws athena start-query-execution \
-    --query-string "${query}" \
-    --result-configuration "OutputLocation=${AWS_ATHENA_OUTPUT_LOCATION}" \
-    --region us-east-1 | jq -r ".QueryExecutionId"
-)
-
-echo "QueryExecutionId is ${queryExecutionId}"
-for i in $(seq 1 120); do
-  queryState=$(
-    aws athena get-query-execution \
-      --query-execution-id "${queryExecutionId}" \
-      --region us-east-1 | jq -r ".QueryExecution.Status.State"
-  )
-
-  if [[ "${queryState}" == "SUCCEEDED" ]]; then
-    break
-  fi
-
-  echo "QueryExecutionId ${queryExecutionId} - state is ${queryState}"
-
-  if [[ "${queryState}" == "FAILED" ]]; then
-    exit 1
-  fi
-
-  sleep 2
-done
-
-echo "Query succeeded. Processing data"
-queryResult=$(
-  aws athena get-query-results \
-    --query-execution-id "${queryExecutionId}" \
-    --region us-east-1 | jq --compact-output
-)
-
-! read -r -d '' jsonTemplate <<EOM
-{
-"type": "gauge",
-"name": "arduino.downloads.total",
-"value": "%s",
-"host": "${GITHUB_REPOSITORY}",
-"tags": [
-"version:%s",
-"os:%s",
-"arch:%s",
-"cdn:downloads.arduino.cc",
-"project:arduino-ide"
-]
-},
-EOM
-
-datapoints="["
-for row in $(echo "${queryResult}" | jq 'del(.ResultSet.Rows[0])' | jq -r '.ResultSet.Rows[] | .Data' --compact-output); do
-  value=$(jq -r ".[1].VarCharValue" <<<"${row}")
-  tag=$(jq -r ".[0].VarCharValue" <<<"${row}")
-  # Some splitting to obtain 0.6.0, Windows, 32bit elements from string 0.6.0_Windows_32bit.zip
-  split=($(echo "$tag" | tr '_' '\n'))
-  if [[ ${#split[@]} -ne 3 ]]; then
-    continue
-  fi
-  archSplit=($(echo "${split[2]}" | tr '.' '\n'))
-  datapoints+=$(printf "${jsonTemplate}" "${value}" "${split[0]}" "${split[1]}" "${archSplit[0]}")
-done
-datapoints="${datapoints::-1}]"
-
-echo "::set-output name=result::$(jq --compact-output <<<"${datapoints}")"

.github/workflows/arduino-stats.yaml

@@ -18,6 +18,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
       - name: Fetch downloads count form Arduino CDN using AWS Athena
         id: fetch
         env:
@@ -27,11 +31,8 @@ jobs:
           AWS_ATHENA_OUTPUT_LOCATION: ${{ secrets.STATS_AWS_ATHENA_OUTPUT_LOCATION }}
           GITHUB_REPOSITORY: ${{ github.repository }}
         run: |
-          # Fetch jq 1.6 as VM has only 1.5 ATM
+          pip install boto3 semver
-          wget -q https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O jq
+          python .github/tools/fetch_athena_stats.py
-          chmod +x jq
-          PATH="${{ github.workspace }}:$PATH"
-          .github/tools/fetch_athena_stats.sh
 
       - name: Send metrics
         uses: masci/datadog@v1