diff --git a/tools/distro-tool b/tools/distro-tool new file mode 100755 index 0000000000..d9150b031e --- /dev/null +++ b/tools/distro-tool @@ -0,0 +1,383 @@ +#!/bin/bash +################################################################################ +# This file is part of LibreELEC - http://www.libreelec.tv +# Copyright (C) 2009-2016 Team LibreELEC +# +# LibreELEC is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# LibreELEC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with LibreELEC. If not, see . +################################################################################ +set -e + +LIBREELEC_DIR=$HOME/LibreELEC.tv +TARGET_DIR=/var/www/sources +DOWNLOAD_DIR=$HOME/download +REVISION= +PACKAGE= +DRYRUN=no +IGNORE_ERRORS=no +DOGIT=no +PROGRESS=yes +ISMIRROR=yes +VERBOSE=no + +[ -z "${PROJECT}" ] && PROJECT=Generic +[ -z "${ARCH}" ] && ARCH=x86_64 + +if command -v tput >/dev/null; then + TXRED="$(tput setaf 1 bold)" + TXGREEN="$(tput setaf 2 bold)" + TXYELLOW="$(tput setaf 3 bold)" + TXBLUE="$(tput setaf 4 bold)" + TXMAGENTA="$(tput setaf 5 bold)" + TXCYAN="$(tput setaf 6 bold)" + TXRESET="$(tput sgr0)" +fi + +help() { + [ -n "$1" ] && echo -e "ERROR: Unknown argument [$1]\n" + cat < [-t|--target ] [-l|--libreelec ] + [-m|--mirror] [-s|--source] [-a|-all] [-p|--package [-r|--revision ]] + [--git] [--dry-run] [--noprogress] [-v|--verbose] [-h|--help] + +Options: + -d, --download: Directory path into which new package files will be downloaded - default is $HOME/download[1] + -t, --target: Directory path for existing packages that are to be refreshed, default is /var/www/sources[2] + -l, --libreelec: LibreELEC.tv repo, default is ${HOME}/LibreELEC.tv + -m, --mirror: Target is mirror not source - mirror uses a hierarchical per-package folder structure + -s, --source: Target is source not mirror - source uses a flattened file structure. Default is mirror + -a, --all: Ignore download failures, continue processing all packages + -p, --package: Package to process, otherwise process all packages + -r, --revision: Version to use in place of PKG_VERSION, only applicable in conjunction with -p + --git: Clone (if not available) or pull the LibreELEC.tv repository + --dry-run: Don't actually download anything (will still git clone/pull if configured) + --noprogress: Do not show progress indicator + -v, --verbose: Output more verbose sync information + -h, --help: This message + + Note#1. The download directory will have the LibreELEC version appended (eg. /devel) unless it is a mirror, in which case "/mirror" will be appended. + Note#2. The target directory will have the LibreELEC version appended (eg. /devel) unless it is a mirror, in which case "/mirror" will be appended. +EOF +} + +get_libreelec_branch() { + cd $LIBREELEC_DIR + git rev-parse --abbrev-ref HEAD +} + +# 1: LibreELEC variable, eg. LIBREELEC_VERSION +get_libreelec_option() { + local variable="$1" + cd $LIBREELEC_DIR + . config/options + echo "${!variable}" +} + +# 1: name of package +# 2: package variable, eg. PKG_URL +get_pkg_option() { + local package="$1" variable="$2" + cd $LIBREELEC_DIR + . config/options ${package} 2>/dev/null + echo "${!variable}" +} + +# Return success if the specified package is not found locally +need_package() { + local package_name="$1" + local package_source="$2" + + if [ ${ISMIRROR} == no ]; then + [ -f ${TARGET_DIR}/${package_source} ] && return 1 + [ -f ${DOWNLOAD_DIR}/${package_source} ] && return 1 + else + [ -f ${TARGET_DIR}/${package_name}/${package_source} ] && return 1 + [ -f ${DOWNLOAD_DIR}/${package_name}/${package_source} ] && return 1 + fi + + return 0 +} + +remote_file_exists() { + [ "$(curl -sfIL --connect-timeout 15 --retry 3 -w "%{http_code}" -o /dev/null ${1} 2>/dev/null)" == "404" ] && return 1 || return 0 +} + +download_file() { + local url="$1" + local retries=10 + local attempts=0 + + while [ ${attempts} -lt ${retries} ]; do + attempts=$((attempts + 1)) + wget --timeout=30 --tries=3 --passive-ftp --no-check-certificate -O ${DOWNLOAD_DIR}/.tmp ${url} 2>/tmp/sync.log && return 0 + done + return 1 +} + +# Download the specified package +getkpkg() { + local package_name="$1" + local package_source="$2" + local package_url="$3" + local result + local onsource=no onmirror=no + + [ -n "${package_url}" ] || return 0 + + remote_file_exists ${package_url} && onsource=yes + if [ ${onsource} == no ]; then + remote_file_exists ${DISTRO_MIRROR}/${package_name}/${package_source} && onmirror=yes + fi + + # If the PKG_URL is the DISTRO_SRC server... + if [[ ${package_url} =~ ^${DISTRO_SOURCE}.* ]]; then + # Warn the user if package is not found on either source or mirror + if [ ${onsource} == no -a ${onmirror} == no ]; then + echo "${LINE_PREFIX}${TXMAGENTA}PACKAGE UPDATE OR MKPKG REQUIRED?${TXRESET}: Unable to download ${package_name} from url ${package_url} (and not on mirror)" >&2 + return 0 + fi + fi + + if [ ${DRYRUN} == yes ]; then + echo -n "${LINE_PREFIX}Downloading package: ${package_name} (${package_source})..." + [ ${onsource} == yes -o ${onmirror} == yes ] && echo -n " ${TXGREEN}AVAILABLE${TXRESET}" || echo -n " ${TXRED}NOT AVAILABLE${TXRESET} ${package_url}" + [ ${onsource} == no -a ${onmirror} == yes ] && echo " ${TXYELLOW}(found on mirror)${TXRESET}" || echo + return 0 + fi + + rm -f ${DOWNLOAD_DIR}/.tmp /tmp/sync.log + + echo -n "${LINE_PREFIX}Downloading package: ${package_name} (${package_source})..." + + if [ ${onsource} == yes ]; then + download_file ${package_url} && result=0 || result=1 + elif [ ${onmirror} == yes ]; then + download_file ${DISTRO_MIRROR}/${package_name}/${package_source} && result=0 || result=1 + else + result=1 + fi + + if [ ${result} -ne 0 -o ! -s ${DOWNLOAD_DIR}/.tmp ]; then + echo " ${TXRED}FAILED!!${TXRESET}" + [ -f /tmp/sync.log ] && cat /tmp/sync.log >&2 + rm -f ${DOWNLOAD_DIR}/.tmp + else + echo " ${TXGREEN}OK${TXRESET}" + if [ ${ISMIRROR} == yes ]; then + mkdir -p ${DOWNLOAD_DIR}/${package_name} + mv ${DOWNLOAD_DIR}/.tmp ${DOWNLOAD_DIR}/${package_name}/${package_source} + else + mv ${DOWNLOAD_DIR}/.tmp ${DOWNLOAD_DIR}/${package_source} + fi + fi + + [ ${IGNORE_ERRORS} == yes ] && return 0 || return ${result} +} + +check_single_package() { + local package_name="$1" revision="$2" + local package_dir="$(cd ${LIBREELEC_DIR}; find packages -type d -name ${PACKAGE})" + + if [ -n "${package_dir}" ]; then + if [ -f ${LIBREELEC_DIR}/${package_dir}/package.mk ]; then + check_package "${package_dir}/package.mk" "${revision}" + return 0 + else + echo "ERROR: ${package_name} is not a valid package - package.mk does not exist" + fi + else + echo "ERROR: ${package_name} is not a valid package - directory does not exist" + fi + return 1 +} + +# Download a package if required +check_package() { + local package_file="$1" revision="$2" + local package_name="$(basename $(dirname ${package_file}))" + local package_source package_url package_ver + + package_source="$(get_pkg_option "${package_name}" PKG_SOURCE_NAME)" + if [ -n "${revision}" ]; then + package_ver=$(get_pkg_option "${package_name}" PKG_VERSION) + package_source="${package_source/${package_ver}/${revision}}" + fi + + [ -n "${package_source}" ] || return 0 + + if need_package "${package_name}" "${package_source}"; then + package_url="$(get_pkg_option "${package_name}" PKG_URL)" + if [ -n "${revision}" ]; then + package_url="${package_url/${package_ver}/${revision}}" + fi + getkpkg "${package_name}" "${package_source}" "${package_url}" + else + [ ${VERBOSE} == yes ] && echo "${LINE_PREFIX}Already downloaded : ${package_name} (${package_source}) ${TXGREEN}OK${TXRESET}" || true + fi +} + +check_exists() { + if [ -z "${!1}" ]; then + echo "ERROR: ${1} must not be undefined" + return 1 + fi + + [ -d ${!1} ] && return 0 + + echo "ERROR: ${1} ${!1} does not exist" + return 1 +} + +get_abs_path() { + echo "$(readlink -f $1)" +} + +get_package_path() { + echo "$(basename "$(dirname "$0")") $0" +} + +get_packages() { + export -f get_package_path + cd $LIBREELEC_DIR + find packages -name package.mk -exec bash -c get_package_path "{}" \; | sort -k1 | cut -d' ' -f2- +} + +COUNT=0 +progress() { + COUNT=$((COUNT + 1)) + printf "Working... %3d%%\r" $((COUNT * 100 / $1)) +} + +while [ : ]; do + [ -z "$1" ] && break + case $1 in + -d|--download) + shift + DOWNLOAD_DIR=$1 + ;; + -l|--libreelec) + shift + LIBREELEC_DIR=$1 + ;; + -t|--target) + shift + TARGET_DIR=$1 + ;; + -a|--all) + IGNORE_ERRORS=yes + ;; + -p|--package) + shift + PACKAGE=$1 + VERBOSE=yes + ;; + -r|--revision) + shift + REVISION=$1 + ;; + -m|--mirror) + ISMIRROR=yes + ;; + -s|--source) + ISMIRROR=no + ;; + --dry-run|--dryrun) + DRYRUN=yes + LINE_PREFIX="**DRY-RUN** " + ;; + --git) + DOGIT=yes + ;; + -v|--verbose) + VERBOSE=yes + ;; + --noprogress) + PROGRESS=no + ;; + -h|--help) + help + exit 0 + ;; + *) + help "$1" + exit 0 + ;; + esac + shift +done + +if [ ${DOGIT} == yes ]; then + ( + if [ -d ${LIBREELEC_DIR}/.git ]; then + cd ${LIBREELEC_DIR} + git pull + else + mkdir -p $(dirname "${LIBREELEC_DIR}") 2>/dev/null + cd $(dirname "${LIBREELEC_DIR}") + git clone https://github.com/LibreELEC/LibreELEC.tv.git $(basename "${LIBREELEC_DIR}") + fi + ) +fi + +check_exists LIBREELEC_DIR || exit +LIBREELEC_DIR="$(get_abs_path "${LIBREELEC_DIR}")" + +DISTRO_SOURCE="$(get_libreelec_option DISTRO_SRC)" +DISTRO_MIRROR="$(get_libreelec_option DISTRO_MIRROR)" +LIBREELEC_VER="$(get_libreelec_option LIBREELEC_VERSION)" + +if [ ${ISMIRROR} == no ]; then + TARGET_DIR="$(get_abs_path "${TARGET_DIR}/${LIBREELEC_VER}")" +else + TARGET_DIR="$(get_abs_path "${TARGET_DIR}/mirror")" +fi +check_exists TARGET_DIR || exit + +if [ ${ISMIRROR} == no ]; then + DOWNLOAD_DIR="$(get_abs_path "${DOWNLOAD_DIR}/${LIBREELEC_VER}")" +else + DOWNLOAD_DIR="$(get_abs_path "${DOWNLOAD_DIR}/mirror")" +fi +check_exists DOWNLOAD_DIR || exit + +if [ -n "${REVISION}" -a -z "${PACKAGE}" ]; then + echo "ERROR: A single package must be specified with custom revision" + exit 1 +fi + +echo +if [ ${ISMIRROR} == yes ]; then + echo "Synchronising LibreELEC.tv (branch: $(get_libreelec_branch), version: ${LIBREELEC_VER}) with MIRROR server ${TARGET_DIR}" +else + echo "Synchronising LibreELEC.tv (branch: $(get_libreelec_branch), version: ${LIBREELEC_VER}) with SOURCE server ${TARGET_DIR}" +fi +echo +echo "Distro Source is: ${DISTRO_SOURCE}" +echo "Distro Mirror is: ${DISTRO_MIRROR}" +echo " Syncing against: ${TARGET_DIR}" +echo " Downloading to: ${DOWNLOAD_DIR}" +echo " Dry run: ${DRYRUN^}" +echo + +[ -z "${NODELAY}" ] && echo -n "Sync starts in 5 seconds..." && sleep 5 && echo -en "\n\n" + +if [ -n "${PACKAGE}" ]; then + check_single_package "${PACKAGE}" "${REVISION}" +else + packages="$(get_packages)" + pcount="$(echo "${packages}" | wc -l)" + for package_mk in ${packages}; do + [ ${PROGRESS} == yes ] && progress ${pcount} + check_package ${package_mk} + done +fi diff --git a/tools/download-tool b/tools/download-tool new file mode 100755 index 0000000000..a17dfa56c3 --- /dev/null +++ b/tools/download-tool @@ -0,0 +1,46 @@ +#!/bin/bash + +################################################################################ +# This file is part of LibreELEC - http://www.libreelec.tv +# Copyright (C) 2009-2016 Lukas Rusak (lrusak@libreelec.tv) +# +# LibreELEC is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# LibreELEC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with LibreELEC. If not, see . +################################################################################ + +if [ -z "$PROJECT" -a -z "$ARCH" ]; then + echo "Usage: PROJECT= ARCH= $0" + exit 0 +fi + +help() { + echo "Usage: PROJECT= ARCH= $0 [-a|--all]" + echo "Options:" + echo " -a, --all: download all packages including addons" +} + +case $1 in + -a|--all) + ALL_PACKAGES="true" + ;; + -h|--help) + help + exit 0 +esac + +for package in $(find packages/ -name package.mk); do + if [ -n "$(echo $package | grep addons)" -a -z "$ALL_PACKAGES" ]; then + continue + fi + ./scripts/get $(basename `dirname $package`) +done diff --git a/tools/repo-tool b/tools/repo-tool new file mode 100755 index 0000000000..e00ba4992c --- /dev/null +++ b/tools/repo-tool @@ -0,0 +1,152 @@ +#!/bin/bash + +################################################################################ +# This file is part of LibreELEC - http://www.libreelec.tv +# Copyright (C) 2009-2016 Lukas Rusak (lrusak@libreelec.tv) +# +# LibreELEC is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# LibreELEC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with LibreELEC. If not, see . +################################################################################ + +. config/options + +update_addons_xml() { + echo "[*] cleanup addons ..." + olddir="" + find target/addons/$ADDON_VERSION -iname 'changelog*.txt' | sort -rV | while read line ; do + dir=$(dirname $line) + if [ "$olddir" = "$dir" ] ; then + rm -f $line + fi + olddir=$dir + done + + olddir="" + find target/addons/$ADDON_VERSION -iname '*.zip' | sort -rV | while read line ; do + dir=$(dirname $line) + if [ "$olddir" = "$dir" ] ; then + rm -f $line + fi + olddir=$dir + done + + echo "[*] updating addons.xml* ..." + rm -rf .addons + pwd=`pwd` + find target/addons/$ADDON_VERSION -iname addons.xml | while read line ; do + localdir=`echo $line | sed s/addons.xml//g` + echo " [*] updating $line..." + echo ' + +' > $line.tmp + for zip in $localdir/*/*.zip ; do + mkdir -p ".addons/$localdir" + unzip $zip "*/addon.xml" -d ".addons/$localdir" &>/dev/null + done + find .addons/$localdir -iname addon.xml | grep -v resources/ | while read xml ; do + cat $xml | grep -v "> $line.tmp + done + echo ' +' >> $line.tmp + mv $line.tmp $line + cd $localdir + + md5sum addons.xml > addons.xml.md5 + cd $pwd + done + rm -rf .addons +} + +touch_addons_xml() { + for PROJECT in Generic RPi2 WeTek_Play; do + for archfile in projects/$PROJECT/linux/linux.*.conf ; do + ARCH=`echo $archfile | sed -n '$s/\.conf//;$s/.*\.//p'` + if [ ! -d target/addons/$ADDON_VERSION/$PROJECT/$ARCH ]; then + break + fi + if [ ! -f target/addons/$ADDON_VERSION/$PROJECT/$ARCH/addons.xml ]; then + touch target/addons/$ADDON_VERSION/$PROJECT/$ARCH/addons.xml + fi + done + done +} + +upload() { + if [ -f .work/repoconfig ] ; then + . .work/repoconfig + fi + if [ -z "$RSYNC_REPO" ] ; then + echo "*** ERROR: \$RSYNC_REPO not set. see .work/repoconfig ***" + exit 0 + fi + touch_addons_xml + update_addons_xml + rsync -av --progress --delete target/addons/$ADDON_VERSION $RSYNC_REPO +} + +build() { + for PROJECT in $(ls -1 projects); do + for archfile in projects/$PROJECT/linux/linux.*.conf ; do + ARCH=`echo $archfile | sed -n '$s/\.conf//;$s/.*\.//p'` + for package in $(find $1* -iname package.mk) ; do + ( + . $package + if [ "$PKG_IS_ADDON" = "yes" ] ; then + ADDON=$PKG_NAME + PROJECT=$PROJECT ARCH=$ARCH ./scripts/create_addon $ADDON + fi + ) + done + done + done +} + +update_revision() { + for package in $(find $1 -iname package.mk) ; do + ( + . $package + if [ "$PKG_IS_ADDON" = "yes" ] ; then + sed -i -e "s|PKG_REV=.*|PKG_REV=\"$((PKG_REV+1))\"|" $package + fi + ) + done +} + +usage() { + echo " usage: $0 -u to upload" + echo " $0 -b [binary|official|unofficial] to build" + echo " $0 -ru [official|unofficial] to update PKG_REV" + echo " $0 -xml to update the addons.xml" +} + +repo="packages/mediacenter/kodi-binary-addons/game.libretro." + +case $1 in + -b) + build $repo + ;; + -u) + upload + ;; + -ru) + update_revision $repo + ;; + -xml) + touch_addons_xml + update_addons_xml + ;; + *) + usage + ;; +esac +