From 32adf7fe452a67da52d08fa292e42c29e2029a91 Mon Sep 17 00:00:00 2001 From: Portisch Date: Mon, 20 Sep 2021 12:55:51 +0200 Subject: [PATCH 1/4] update_common_functions: allow assigning PROJECT and ARCH values --- tools/mkpkg/update_common_functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/mkpkg/update_common_functions b/tools/mkpkg/update_common_functions index 41ac0708c7..9be63de908 100644 --- a/tools/mkpkg/update_common_functions +++ b/tools/mkpkg/update_common_functions @@ -5,7 +5,7 @@ msg_color() { echo $( cd "${ROOT}" - PROJECT=Generic ARCH=x86_64 . config/options "" + PROJECT="${PROJECT:-Generic}" ARCH="${ARCH:-x86_64}" . config/options "" echo $(print_color "$1" "$2") ) } @@ -42,7 +42,7 @@ git_clone() { get_pkg_var() { local pkg_name="$1" pkg_var="$2" cd "${ROOT}" - PROJECT=Generic ARCH=x86_64 source config/options ${pkg_name} &>/dev/null + PROJECT="${PROJECT:-Generic}" ARCH="${ARCH:-x86_64}" source config/options ${pkg_name} &>/dev/null echo "${!pkg_var}" } From 18488b0f0e7145bafe4b53ed2e11f1a5a66b64b1 Mon Sep 17 00:00:00 2001 From: Matthias Reichl Date: Fri, 24 Sep 2021 13:28:35 +0200 Subject: [PATCH 2/4] update_retroplayer-addons: support tar.gz archive links game addons have (mostly?) switched from zip to tar.gz links, add support for extracting the version info from these as well to fix game addon updates. Signed-off-by: Matthias Reichl --- tools/mkpkg/update_retroplayer-addons | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/mkpkg/update_retroplayer-addons b/tools/mkpkg/update_retroplayer-addons index 428f08c273..463338ef45 100755 --- a/tools/mkpkg/update_retroplayer-addons +++ b/tools/mkpkg/update_retroplayer-addons @@ -160,6 +160,9 @@ for addon in ${ADDONS_DIR}/*.*/ ; do if [[ "$VERSION_INFO" =~ .zip$ ]] ; then # version referenced by githash RETRO_NEW_VERSION=$(sed -e 's|^.*/archive/||' -e 's|\.zip$||' "${RETRO_VERSION_FILE}") + elif [[ "$VERSION_INFO" =~ .tar.gz$ ]] ; then + # version referenced by githash + RETRO_NEW_VERSION=$(sed -e 's|^.*/archive/||' -e 's|\.tar.gz$||' "${RETRO_VERSION_FILE}") else msg_warn "unmanaged version in kodi package: ${VERSION_INFO}" # unmanaged version, repo plus branch From bec6d170546b58f5e6b10690a95096d1fd09efa5 Mon Sep 17 00:00:00 2001 From: Portisch Date: Wed, 22 Sep 2021 11:33:34 +0200 Subject: [PATCH 3/4] update binary addons scripts: implement GITHUB_API_TOKEN alternative When environment variable GITHUB_API_TOKEN like in ${HOME}/.${DISTRO,,}/options is assigned the script will fetch all data direct from Github instead clone every single package. --- tools/mkpkg/update_binary-addons | 64 ++++++++++++++++++++++----- tools/mkpkg/update_common_functions | 29 +++++++++++- tools/mkpkg/update_retroplayer-addons | 23 ++++++++-- 3 files changed, 101 insertions(+), 15 deletions(-) diff --git a/tools/mkpkg/update_binary-addons b/tools/mkpkg/update_binary-addons index 5fd1b67dd4..196db818c2 100755 --- a/tools/mkpkg/update_binary-addons +++ b/tools/mkpkg/update_binary-addons @@ -68,10 +68,33 @@ fi . "${MY_DIR}/update_common_functions" -git_clone https://github.com/xbmc/xbmc ${KODI_DIR} ${KODI_BRANCH} +get_gh_token + +if [ -z "${GITHUB_API_TOKEN}" ]; then + git_clone https://github.com/xbmc/xbmc ${KODI_DIR} ${KODI_BRANCH} + ADDONS_REPO_LOCATION="${KODI_DIR}/cmake/addons/bootstrap/repositories" +else + # check if depends are available + command -v curl >/dev/null 2>&1 || die "please install curl" + command -v jq >/dev/null 2>&1 || die "please install jq" + + TMP_REPO_DIR="${TMPDIR}/bootstrap-repos" + + curl -s -L -H "Authorization: token ${GITHUB_API_TOKEN}" \ + "https://api.github.com/repos/xbmc/xbmc/contents/cmake/addons/bootstrap/repositories?ref=${KODI_BRANCH}" | awk \ + '/download_url/ { gsub("\"|,", "", $2); system("mkdir -p '${TMP_REPO_DIR}'; (cd '${TMP_REPO_DIR}'; curl -O "$2")"); }' \ + 2>/dev/null + ADDONS_REPO_LOCATION=${TMP_REPO_DIR} +fi + +if ! ls ${ADDONS_REPO_LOCATION}/*-addons.txt &> /dev/null; then + echo "No binary addon repo description found, nothing to do!" + echo "Path searched: ${ADDONS_REPO_LOCATION}" + exit 0 +fi # addons -for addontxt in $KODI_DIR/cmake/addons/bootstrap/repositories/*-addons.txt ; do +for addontxt in ${ADDONS_REPO_LOCATION}/*-addons.txt ; do ADDONS=$(cat $addontxt | awk '{print $1}') ADDONS_GIT_DIR="${ADDONS}.git" ADDONS_GIT_REPO=$(cat $addontxt | awk '{print $2}') @@ -99,16 +122,26 @@ for addontxt in $KODI_DIR/cmake/addons/bootstrap/repositories/*-addons.txt ; do # If different, ignore the addon and process it later as an "unofficial" addon validate_pkg_url "${ADDON}" "${GIT_REPO}" || continue - git_clone ${GIT_REPO} ${GIT_DIR} ${GIT_BRANCH} - NO_TAG="" - NEW_VERSION=$(resolve_tag_in_branch ${GIT_DIR} ${GIT_BRANCH}) + if [ -z "${GITHUB_API_TOKEN}" ]; then + git_clone ${GIT_REPO} ${GIT_DIR} ${GIT_BRANCH} + PARAMS="resolve_tag_in_branch ${GIT_DIR} ${GIT_BRANCH}" + else + PARAMS="resolve_tag_on_gh ${GIT_REPO} ${GIT_BRANCH}" + fi + NEW_VERSION=$(${PARAMS}) + if [ -z "${NEW_VERSION}" ]; then NO_TAG="yes" echo "========================================================================" msg_warn "WARNING: no tag found for addon ${ADDON}, falling back to HEAD" echo "========================================================================" - NEW_VERSION=$(resolve_hash_in_branch "${GIT_DIR}" "${GIT_BRANCH}") + if [ -z "${GITHUB_API_TOKEN}" ]; then + PARAMS="resolve_hash_in_branch ${GIT_DIR} ${GIT_BRANCH}" + else + PARAMS="resolve_hash_on_gh ${GIT_REPO} ${GIT_BRANCH}" + fi + NEW_VERSION=$(${PARAMS}) fi echo "Resolved version for ${ADDON}: ${GIT_BRANCH} => ${NEW_VERSION}" @@ -125,7 +158,7 @@ for addontxt in $KODI_DIR/cmake/addons/bootstrap/repositories/*-addons.txt ; do fi if [ "${KEEP_GIT_DIRS}" != "yes" ]; then - rm -rf "${GIT_DIR}" + [ -d "${GIT_DIR}" ] && rm -rf "${GIT_DIR}" fi else echo "[mkpkg] Skipped $ADDON" @@ -154,11 +187,18 @@ for ADDON in $(ls -1 "${ROOT}/packages/mediacenter/kodi-binary-addons"); do GIT_DIR="${ADDON}.git" GIT_REPO="$(geturl "${ADDON}")" || continue - git_clone ${GIT_REPO} ${GIT_DIR} + if [ -z "${GITHUB_API_TOKEN}" ]; then + git_clone ${GIT_REPO} ${GIT_DIR} + RESOLVE_UNOFFICIAL_BRANCH="resolve_hash_in_branch ${ADDON}.git ${UNOFFICIAL_ADDON_BRANCH}" + RESOLVE_HEAD="resolve_hash_in_branch ${ADDON}.git HEAD" + else + RESOLVE_UNOFFICIAL_BRANCH="resolve_hash_on_gh ${GIT_REPO%.git} ${UNOFFICIAL_ADDON_BRANCH}" + RESOLVE_HEAD="resolve_hash_on_gh ${GIT_REPO%.git} HEAD" + fi - if RESOLVED_HASH=$(resolve_hash_in_branch ${ADDON}.git ${UNOFFICIAL_ADDON_BRANCH}); then + if RESOLVED_HASH=$(${RESOLVE_UNOFFICIAL_BRANCH}); then echo "Resolved hash for ${ADDON}: ${UNOFFICIAL_ADDON_BRANCH} => ${RESOLVED_HASH}" - elif RESOLVED_HASH=$(resolve_hash_in_branch ${ADDON}.git HEAD); then + elif RESOLVED_HASH=$(${RESOLVE_HEAD}); then echo "Resolved hash for ${ADDON}: HEAD => ${RESOLVED_HASH}" else msg_warn "WARNING: Could not resolve hash for ${ADDON}" @@ -173,8 +213,10 @@ for ADDON in $(ls -1 "${ROOT}/packages/mediacenter/kodi-binary-addons"); do fi if [ "${KEEP_GIT_DIRS}" != "yes" ]; then - rm -rf ${GIT_DIR} + [ -d "${GIT_DIR}" ] && rm -rf ${GIT_DIR} fi done rm -rf "${TMPDIR}" + +[ -d "${TMP_REPO_DIR}" ] && rm -rf "${TMP_REPO_DIR}" diff --git a/tools/mkpkg/update_common_functions b/tools/mkpkg/update_common_functions index 9be63de908..89db7c20bf 100644 --- a/tools/mkpkg/update_common_functions +++ b/tools/mkpkg/update_common_functions @@ -60,6 +60,12 @@ resolve_hash_in_branch() { fi } +resolve_hash_on_gh() { + curl -s -L -H "Authorization: token ${GITHUB_API_TOKEN}" \ + -H "Accept: application/vnd.github.VERSION.sha" \ + "${1/*github.com/https:\/\/api.github.com\/repos}/commits/$2" +} + resolve_tag_in_branch() { local tag if [ -d "$1" ] ; then @@ -75,6 +81,22 @@ resolve_tag_in_branch() { fi } +resolve_tag_on_gh() { + local tag + + tag=$(curl -s -L -H "Authorization: token ${GITHUB_API_TOKEN}" \ + "${1/*github.com/https:\/\/api.github.com\/repos}/releases" | \ + jq -r '[.[] | select(.target_commitish == "'$2'")][0].tag_name | select (.!=null)') + + if [ -z "$tag" ] ; then + tag=$(curl -s -L -H "Authorization: token ${GITHUB_API_TOKEN}" \ + "${1/*github.com/https:\/\/api.github.com\/repos}/tags" | \ + jq -r '[.[] | select(.name | contains("'$2'"))][0].name | select (.!=null)') + fi + + echo "$tag" +} + check_package_excluded() { local package="$1" local packages_to_exclude="$2" pkg [ -z "${package}" -o -z "${packages_to_exclude}" ] && return 1 @@ -183,5 +205,10 @@ validate_pkg_url() { [ "${url1}" = "${url2}" ] && return 0 || return 1 } - +# Get user Github token if available +get_gh_token() { + if [ -z "${GITHUB_API_TOKEN}" ]; then + GITHUB_API_TOKEN=$(get_pkg_var "" GITHUB_API_TOKEN) + fi +} diff --git a/tools/mkpkg/update_retroplayer-addons b/tools/mkpkg/update_retroplayer-addons index 463338ef45..c4944e1567 100755 --- a/tools/mkpkg/update_retroplayer-addons +++ b/tools/mkpkg/update_retroplayer-addons @@ -73,6 +73,14 @@ mkdir -p "${TMPDIR}" . "${MY_DIR}/update_common_functions" +get_gh_token + +# check if depends are available +if [ -n "${GITHUB_API_TOKEN}" ]; then + command -v curl >/dev/null 2>&1 || die "please install curl" + command -v jq >/dev/null 2>&1 || die "please install jq" +fi + ADDONS="game-binary-addons" ADDONS_DIR="${ADDONS}.git" ADDONS_REPO="https://github.com/kodi-game/repo-binary-addons.git" @@ -114,14 +122,23 @@ for addon in ${ADDONS_DIR}/*.*/ ; do NO_TAG="" CHECK_RETRO="" - git_clone "${GAME_GIT_REPO}" "${GAME_GIT_DIR}" - GAME_NEW_VERSION=$(resolve_tag_in_branch "${GAME_GIT_DIR}" "${GAME_GIT_BRANCH}" "*-${KODI_BRANCH}") + if [ -z "$GITHUB_API_TOKEN" ]; then + git_clone "${GAME_GIT_REPO}" "${GAME_GIT_DIR}" + GAME_NEW_VERSION=$(resolve_tag_in_branch "${GAME_GIT_DIR}" "${GAME_GIT_BRANCH}" "*-${KODI_BRANCH}") + else + GAME_NEW_VERSION=$(resolve_tag_on_gh "${GAME_GIT_REPO}" "-${KODI_BRANCH}") + fi + if [ -z "${GAME_NEW_VERSION}" ]; then NO_TAG="yes" echo "========================================================================" msg_warn "WARNING: no tag found for addon ${GAME_ADDON}, falling back to HEAD" echo "========================================================================" - GAME_NEW_VERSION=$(resolve_hash_in_branch "${GAME_GIT_DIR}" "${GAME_GIT_BRANCH}") + if [ -z "$GITHUB_API_TOKEN" ]; then + GAME_NEW_VERSION=$(resolve_hash_in_branch "${GAME_GIT_DIR}" "${GAME_GIT_BRANCH}") + else + GAME_NEW_VERSION=$(resolve_hash_on_gh "${GAME_GIT_REPO}" "${GAME_GIT_BRANCH}") + fi fi if [ "${KEEP_GIT_DIRS}" != "yes" ]; then From eecccee3c7d2feb55e00da7970f728c4df89433d Mon Sep 17 00:00:00 2001 From: Portisch Date: Mon, 4 Oct 2021 11:33:02 +0200 Subject: [PATCH 4/4] update_binary-addons: remove ADDONS_GIT_DIR if delete git dirs is set --- tools/mkpkg/update_binary-addons | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/mkpkg/update_binary-addons b/tools/mkpkg/update_binary-addons index 196db818c2..fa9f2fb520 100755 --- a/tools/mkpkg/update_binary-addons +++ b/tools/mkpkg/update_binary-addons @@ -167,6 +167,10 @@ for addontxt in ${ADDONS_REPO_LOCATION}/*-addons.txt ; do PROCESSED="${PROCESSED}${ADDON}\n" done + if [ "${KEEP_GIT_DIRS}" != "yes" ]; then + [ -d "${ADDONS_GIT_DIR}" ] && rm -rf "${ADDONS_GIT_DIR}" + fi + echo echo "The following addons were skipped, please add packages for these addons:" echo "[${SKIPPED_ADDONS}]"