Merge pull request #2529 from MilhouseVH/le90_get_handlers

buildsystem: support get handlers (archive, file, git)
This commit is contained in:
CvH 2018-04-02 11:02:52 +02:00 committed by GitHub
commit 42f3a2f5a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 426 additions and 190 deletions

View File

@ -98,6 +98,13 @@ if [ -n "$PKG_URL" -a -z "$PKG_SOURCE_NAME" ]; then
echo "$PKG_URL"
exit 1
fi
if [[ ${PKG_URL} =~ .git$ || ${PKG_URL} =~ ^git:// ]]; then
PKG_SOURCE_NAME=${PKG_NAME}-${PKG_VERSION}
elif [[ ${PKG_URL} =~ ^file:// ]]; then
PKG_SOURCE_NAME=${PKG_URL#file://}
# if no specific PKG_TAR_COPY_OPTS then default to excluding .git and .svn as they can be huge
[ -z "${PKG_TAR_COPY_OPTS+x}" ] && PKG_TAR_COPY_OPTS="--exclude=.git --exclude=.svn"
else
PKG_SOURCE_NAME="${PKG_URL##*/}"
case $PKG_SOURCE_NAME in
${PKG_NAME}-${PKG_VERSION}.*)
@ -117,6 +124,7 @@ if [ -n "$PKG_URL" -a -z "$PKG_SOURCE_NAME" ]; then
;;
esac
fi
fi
PKG_BUILD="$BUILD/${PKG_NAME}-${PKG_VERSION}"

View File

@ -207,3 +207,6 @@
# Configure debug groups (space delimited key=value pairs, with each value comma-delimited) and default group when DEBUG=yes
DEBUG_GROUPS="kodi=kodi,kodi-platform,p8-platform,!mesa"
DEBUG_GROUP_YES="kodi"
# Default supported get handlers (archive, git, file etc.)
GET_HANDLER_SUPPORT="archive"

View File

@ -28,47 +28,51 @@ fi
[ -z "$PKG_URL" -o -z "$PKG_SOURCE_NAME" ] && exit 1
[ ! -d "$SOURCES/$1" -o ! -d "$2" ] && exit 1
for pattern in .tar.gz .tar.xz .tar.bz2 .tgz .txz .tbz .7z .zip; do
if [[ $PKG_SOURCE_NAME =~ ${pattern//./\\.}$ ]]; then
f="$SOURCES/$1/$PKG_SOURCE_NAME"
if [ ! -f $f ]; then
echo "error: File $PKG_SOURCE_NAME doesn't exist in package $1 sources directory"
if [[ ${PKG_URL} =~ ^file:// ]]; then
FULL_SOURCE_PATH="$PKG_SOURCE_NAME"
else
FULL_SOURCE_PATH="$SOURCES/$1/$PKG_SOURCE_NAME"
fi
if [ ! -f "$FULL_SOURCE_PATH" -a ! -d "$FULL_SOURCE_PATH" ]; then
echo "error: File $PKG_SOURCE_NAME doesn't exist for package $1"
echo "Have you called scripts/extract before scripts/get ?"
exit 1
fi
case $PKG_SOURCE_NAME in
*.tar)
tar xf $f -C $2
tar xf $FULL_SOURCE_PATH -C $2
;;
*.tar.bz2 | *.tbz)
tar xjf $f -C $2
tar xjf $FULL_SOURCE_PATH -C $2
;;
*.tar.gz | *.tgz)
tar xzf $f -C $2
tar xzf $FULL_SOURCE_PATH -C $2
;;
*.tar.xz | *.txz)
tar xJf $f -C $2
tar xJf $FULL_SOURCE_PATH -C $2
;;
*.7z)
mkdir -p $2/$1
7z x -o$2/$1 $f
7z x -o$2/$1 $FULL_SOURCE_PATH
;;
*.zip)
unzip -q $f -d $2
unzip -q $FULL_SOURCE_PATH -d $2
;;
*.diff | *.patch)
cat $f | patch -d $2 -p1
cat $FULL_SOURCE_PATH | patch -d $2 -p1
;;
*.diff.bz2 | *.patch.bz2 | patch-*.bz2)
bzcat $f | patch -d $2 -p1
bzcat $FULL_SOURCE_PATH | patch -d $2 -p1
;;
*.diff.gz | *.patch.gz | patch-*.gz)
zcat $f | patch -d $2 -p1
zcat $FULL_SOURCE_PATH | patch -d $2 -p1
;;
*)
cp -pPR $f $2
FULL_DEST_PATH="$2/$PKG_NAME-$PKG_VERSION"
mkdir $FULL_DEST_PATH
tar cf - -C $FULL_SOURCE_PATH $PKG_TAR_COPY_OPTS . | \
tar xf - -C $FULL_DEST_PATH
;;
esac
break
fi
done

View File

@ -20,12 +20,6 @@
. config/options $1
_get_file_already_downloaded() {
[ ! -f $PACKAGE -o ! -f $STAMP_URL -o ! -f $STAMP_SHA ] && return 1
[ -n "${PKG_SHA256}" -a "$(cat $STAMP_SHA 2>/dev/null)" != "${PKG_SHA256}" ] && return 1
return 0
}
if [ -z "$1" ]; then
for i in `find packages/ -type f -name package.mk`; do
GET_PKG=`grep ^PKG_NAME= $i | sed -e "s,\",,g" -e "s,PKG_NAME=,,"`
@ -33,63 +27,46 @@ if [ -z "$1" ]; then
done
fi
[ -z "$PKG_URL" -o -z "$PKG_SOURCE_NAME" ] && exit 0
# Avoid concurrent processing of the same package
function lock_source_dir() {
local _isblocked=N
exec 99<$SOURCES/$1
while ! flock --nonblock --exclusive 99; do
[ ${_isblocked} = N ] && { echo "Project/Device ${DEVICE:-${PROJECT}} waiting, to avoid concurrent processing of ${1}..."; _isblocked=Y; }
sleep 1
done
}
if [ -n "$PKG_URL" -a -n "$PKG_SOURCE_NAME" ]; then
mkdir -p $SOURCES/$1
PACKAGE="$SOURCES/$1/$PKG_SOURCE_NAME"
PACKAGE_MIRROR="$DISTRO_MIRROR/$PKG_NAME/$PKG_SOURCE_NAME"
[ "$VERBOSE" != "yes" ] && WGET_OPT=-q
WGET_CMD="wget --timeout=30 --tries=3 --passive-ftp --no-check-certificate -c $WGET_OPT -O $PACKAGE"
STAMP_URL="$PACKAGE.url"
STAMP_SHA="$PACKAGE.sha256"
# Latest file already present, exit now...
_get_file_already_downloaded $1 && exit 0
# determine get handler based on protocol and/or filename
case "${PKG_URL}" in
git://*|*.git)
get_handler="git";;
file://*)
get_handler="file";;
*)
get_handler="archive";;
esac
# Avoid concurrent downloads of the same package
_isblocked=N
exec 99<$SOURCES/$1
while ! flock --nonblock --exclusive 99; do
[ ${_isblocked} == N ] && { echo "Project/Device ${DEVICE:-${PROJECT}} waiting, to avoid concurrent download of ${1}..."; _isblocked=Y; }
sleep 1
done
# Check again in case of concurrent access - if nothing needs to be downloaded, exit now...
_get_file_already_downloaded $1 && exit 0
# At this point, we need to download something...
printf "%${BUILD_INDENT}c $(print_color CLR_GET "GET") $1\n" ' '>&$SILENT_OUT
export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE))
# unset LD_LIBRARY_PATH to stop wget from using toolchain/lib and loading libssl.so/libcrypto.so instead of host libraries
unset LD_LIBRARY_PATH
rm -f $STAMP_URL $STAMP_SHA
NBWGET=10
while [ $NBWGET -gt 0 ]; do
for url in "$PKG_URL" "$PACKAGE_MIRROR"; do
rm -f $PACKAGE
if $WGET_CMD "$url"; then
CALC_SHA256="$(sha256sum $PACKAGE | cut -d" " -f1)"
[ -z "${PKG_SHA256}" -o "${PKG_SHA256}" == "${CALC_SHA256}" ] && break 2
printf "%${BUILD_INDENT}c $(print_color CLR_WARNING "WARNING") Incorrect checksum calculated on downloaded file: got ${CALC_SHA256}, wanted ${PKG_SHA256}\n\n" ' '>&$SILENT_OUT
fi
done
NBWGET=$((NBWGET - 1))
done
if [ $NBWGET -eq 0 ]; then
echo -e "\nCant't get $1 sources : $PKG_URL\n Try later !!"
if ! listcontains "${GET_HANDLER_SUPPORT}" "${get_handler}"; then
echo "ERROR: get handler \"${get_handler}\" is not supported, unable to get package $1 - aborting!"
exit 1
else
printf "%${BUILD_INDENT}c $(print_color CLR_INFO "INFO") Calculated checksum: ${CALC_SHA256}\n\n" ' '>&$SILENT_OUT
echo "${PKG_URL}" > $STAMP_URL
echo "${CALC_SHA256}" > $STAMP_SHA
get_handler="${SCRIPTS}/get_${get_handler}"
if [ ! -f ${get_handler} ]; then
echo "ERROR: get handler \"${get_handler}\" does not exist, unable to get package $1 - aborting!"
exit 1
else
source ${get_handler}
fi
fi
fi
exit 0

68
scripts/get_archive Executable file
View File

@ -0,0 +1,68 @@
################################################################################
# This file is part of LibreELEC - https://libreelec.tv
# Copyright (C) 2018-present Team LibreELEC
#
# LibreELEC is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# LibreELEC is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with LibreELEC. If not, see <http://www.gnu.org/licenses/>.
################################################################################
_get_file_already_downloaded() {
[ ! -f $PACKAGE -o ! -f $STAMP_URL -o ! -f $STAMP_SHA ] && return 1
[ -n "${PKG_SHA256}" -a "$(cat $STAMP_SHA 2>/dev/null)" != "${PKG_SHA256}" ] && return 1
return 0
}
# Latest file already present, exit now...
_get_file_already_downloaded && exit 0
lock_source_dir $1
# Check again in case of concurrent access - if nothing needs to be downloaded, exit now...
_get_file_already_downloaded && exit 0
# At this point, we need to download something...
printf "%${BUILD_INDENT}c $(print_color CLR_GET "GET") $1 (archive)\n" ' '>&$SILENT_OUT
export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE))
PACKAGE_MIRROR="$DISTRO_MIRROR/$PKG_NAME/$PKG_SOURCE_NAME"
[ "$VERBOSE" != "yes" ] && WGET_OPT=-q
WGET_CMD="wget --timeout=30 --tries=3 --passive-ftp --no-check-certificate -c $WGET_OPT -O $PACKAGE"
# unset LD_LIBRARY_PATH to stop wget from using toolchain/lib and loading libssl.so/libcrypto.so instead of host libraries
unset LD_LIBRARY_PATH
rm -f $STAMP_URL $STAMP_SHA
NBWGET=10
while [ $NBWGET -gt 0 ]; do
for url in "$PKG_URL" "$PACKAGE_MIRROR"; do
rm -f $PACKAGE
if $WGET_CMD "$url"; then
CALC_SHA256="$(sha256sum $PACKAGE | cut -d" " -f1)"
[ -z "${PKG_SHA256}" -o "${PKG_SHA256}" = "${CALC_SHA256}" ] && break 2
printf "%${BUILD_INDENT}c $(print_color CLR_WARNING "WARNING") Incorrect checksum calculated on downloaded file: got ${CALC_SHA256}, wanted ${PKG_SHA256}\n\n" ' '>&$SILENT_OUT
fi
done
NBWGET=$((NBWGET - 1))
done
if [ $NBWGET -eq 0 ]; then
echo -e "\nCant't get $1 sources : $PKG_URL\n Try later !!"
exit 1
else
printf "%${BUILD_INDENT}c $(print_color CLR_INFO "INFO") Calculated checksum: ${CALC_SHA256}\n\n" ' '>&$SILENT_OUT
echo "${PKG_URL}" > $STAMP_URL
echo "${CALC_SHA256}" > $STAMP_SHA
fi

20
scripts/get_file Executable file
View File

@ -0,0 +1,20 @@
################################################################################
# This file is part of LibreELEC - https://libreelec.tv
# Copyright (C) 2018-present Team LibreELEC
#
# LibreELEC is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# LibreELEC is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with LibreELEC. If not, see <http://www.gnu.org/licenses/>.
################################################################################
printf "%${BUILD_INDENT}c $(print_color CLR_GET "GET") $1 (file)\n" ' '>&$SILENT_OUT
export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE))

154
scripts/get_git Executable file
View File

@ -0,0 +1,154 @@
################################################################################
# This file is part of LibreELEC - https://libreelec.tv
# Copyright (C) 2018-present Team LibreELEC
#
# LibreELEC is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# LibreELEC is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with LibreELEC. If not, see <http://www.gnu.org/licenses/>.
################################################################################
# Handler for git
# Usage (in package.mk):
# PKG_URL (mandatory) must point to a git repository (git://... or https://example.com/repo.git)
# PKG_VERSION (mandatory) must point to a commit SHA, e.g. a1b2c3d
# PKG_GIT_SHA (optional) full hash of git commit
# PKG_GIT_CLONE_BRANCH (optional) clone specific branch
# PKG_GIT_CLONE_SINGLE (optional) clone single branch only (set to yes)
# PKG_GIT_CLONE_DEPTH (optional) history to clone, must be a number
# PKG_GIT_SUBMODULE_DEPTH (optional) history of submodules to clone, must be a number
_get_repo_already_downloaded() {
if [ -d $PACKAGE ]; then
(
cd $PACKAGE
_get_repo_clean
[ -n "$(git ls-remote . | grep -m1 HEAD | awk "/^${PKG_VERSION}/ {print \$1;}")" ] || exit 1
[ "$PKG_URL" = "$(git remote get-url origin)" ] || exit 1
[ -z "$PKG_GIT_CLONE_BRANCH" ] && exit 0
[ "$PKG_GIT_CLONE_BRANCH" = "$(git branch | grep ^\* | cut -d ' ' -f2)" ] || exit 1
exit 0
)
return
else
return 1
fi
}
_get_repo_clean() {
git clean -fdx
git checkout -- .
}
# Latest file already present, exit now...
_get_repo_already_downloaded && exit 0
lock_source_dir $1
# Check again in case of concurrent access - if nothing needs to be downloaded, exit now...
_get_repo_already_downloaded && exit 0
# At this point, we need to download something...
printf "%${BUILD_INDENT}c $(print_color CLR_GET "GET") $1 (git)\n" ' '>&$SILENT_OUT
export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE))
rm -f $STAMP_URL $STAMP_SHA
GIT_CLONE_PARAMS=""
GIT_SUBMODULE_PARAMS=""
[ -n "$PKG_GIT_CLONE_BRANCH" ] && GIT_CLONE_PARAMS="$GIT_CLONE_PARAMS --branch $PKG_GIT_CLONE_BRANCH"
[ "$PKG_GIT_CLONE_SINGLE" = "yes" ] && GIT_CLONE_PARAMS="$GIT_CLONE_PARAMS --single-branch"
if [ -n "$PKG_GIT_CLONE_DEPTH" ]; then
if [[ $PKG_GIT_CLONE_DEPTH =~ ^[0-9]+$ ]]; then
GIT_CLONE_PARAMS="$GIT_CLONE_PARAMS --depth $PKG_GIT_CLONE_DEPTH"
else
echo "Fatal: PKG_GIT_CLONE_DEPTH is not a number! ($PKG_GIT_CLONE_DEPTH)"
exit 1
fi
fi
if [ -n "$PKG_GIT_SUBMODULE_DEPTH" ]; then
if [[ $PKG_GIT_SUBMODULE_DEPTH =~ ^[0-9]+$ ]]; then
GIT_SUBMODULE_PARAMS="$GIT_SUBMODULE_PARAMS --depth $PKG_GIT_SUBMODULE_DEPTH"
else
echo "Fatal: PKG_GIT_SUBMODULE_DEPTH is not a number! ($PKG_GIT_SUBMODULE_DEPTH)"
exit 1
fi
fi
GIT_FOUND="no"
opwd=$(pwd)
for d in $SOURCES/$1/$1-* ; do
if [ -d "$d/.git" ]; then
if [ "${GIT_FOUND}" = "no" ]; then
cd $d
if [ "$PKG_URL" = "$(git remote get-url origin)" ]; then
if [ -n "$PKG_GIT_CLONE_BRANCH" -a $(git branch | grep "^\* ${PKG_GIT_CLONE_BRANCH}$" | wc -l) -eq 1 -o -z "$PKG_GIT_CLONE_BRANCH" ]; then
GIT_FOUND="yes"
GIT_DIR="$d"
_get_repo_clean
elif [ -n "$PKG_GIT_CLONE_BRANCH" -a $(git branch | grep "^ ${PKG_GIT_CLONE_BRANCH}$" | wc -l) -eq 1 ]; then
GIT_FOUND="yes"
GIT_DIR="$d"
_get_repo_clean
git checkout $PKG_GIT_CLONE_BRANCH
elif [ -n "$PKG_GIT_CLONE_BRANCH" -a $(git branch -a | grep "^ remotes/origin/${PKG_GIT_CLONE_BRANCH}$" | wc -l) -eq 1 ]; then
GIT_FOUND="yes"
GIT_DIR="$d"
_get_repo_clean
git checkout -b $PKG_GIT_CLONE_BRANCH origin/$PKG_GIT_CLONE_BRANCH
else
printf "%${BUILD_INDENT}c $(print_color CLR_CLEAN "DELETE") ($d)\n" ' '>&$SILENT_OUT
cd "${opwd}"
rm -rf "$d"
fi
if [ "$GIT_FOUND" = "yes" ]; then
printf "%${BUILD_INDENT}c $(print_color CLR_GET "GIT PULL") $1\n" ' '>&$SILENT_OUT
git pull
cd "${opwd}"
fi
else
printf "%${BUILD_INDENT}c $(print_color CLR_CLEAN "DELETE") ($d)\n" ' '>&$SILENT_OUT
cd "${opwd}"
rm -rf "$d"
fi
else
printf "%${BUILD_INDENT}c $(print_color CLR_CLEAN "DELETE") ($d)\n" ' '>&$SILENT_OUT
rm -rf "$d"
fi
fi
done
cd "${opwd}"
if [ "${GIT_FOUND}" = "no" ]; then
printf "%${BUILD_INDENT}c $(print_color CLR_GET "GIT CLONE") $1\n" ' '>&$SILENT_OUT
git clone $GIT_CLONE_PARAMS $PKG_URL $PACKAGE
else
if [ ! "${GIT_DIR}" = "${PACKAGE}" ]; then
mv "${GIT_DIR}" "${PACKAGE}"
fi
fi
(
cd $PACKAGE
[ $(git log --oneline --pretty=tformat:"%H" | grep "^$PKG_VERSION" | wc -l) -eq 1 ] || { echo "There is no commit '$PKG_VERSION' on branch '$(git branch | grep ^\* | cut -d ' ' -f2)' of package '$1'! Aborting!" ; exit 1 ; }
git reset --hard $PKG_VERSION
printf "%${BUILD_INDENT}c $(print_color CLR_GET "GIT SUBMODULE") $1\n" ' '>&$SILENT_OUT
git submodule update --init --recursive $GIT_SUBMODULE_PARAMS
)
GIT_SHA=$(git ls-remote $PACKAGE | grep -m1 HEAD | awk '{print $1;}')
if [ -n "$PKG_GIT_SHA" ]; then
[ "$PKG_GIT_SHA" = "$GIT_SHA" ] || printf "%${BUILD_INDENT}c $(print_color CLR_WARNING "WARNING") Incorrect git hash in respository: got ${GIT_SHA}, wanted ${PKG_GIT_SHA}\n\n" ' '>&$SILENT_OUT
fi

View File

@ -106,6 +106,7 @@ if [ -d "$SOURCES/$1" -o -d "$PKG_DIR/sources" ]; then
post_unpack
fi
if [ "${PKG_SKIP_PATCHES}" != "yes" ]; then
if [ "$(type -t pre_patch)" = "function" ]; then
pre_patch
fi
@ -180,6 +181,7 @@ if [ -d "$SOURCES/$1" -o -d "$PKG_DIR/sources" ]; then
if [ "$(type -t post_patch)" = "function" ]; then
post_patch
fi
fi
if [ ! "$PKG_NAME" == "configtools" ] ; then
for config in `find $PKG_BUILD -name config.guess | sed 's/config.guess//'`; do