Merge pull request #4833 from OpenELEC/Python-fixes

Python fixes
This commit is contained in:
Stephan Raue 2016-03-10 20:18:01 +01:00
commit 0de0a2a898
10 changed files with 341 additions and 90 deletions

View File

@ -22,6 +22,11 @@ setup_toolchain() {
export PKG_CONFIG_LIBDIR="$TARGET_PKG_CONFIG_LIBDIR"
export PKG_CONFIG_SYSROOT_DIR="$TARGET_PKG_CONFIG_SYSROOT_DIR"
# set python defaults
export _python_sysroot="$SYSROOT_PREFIX"
export _python_prefix=/usr
export _python_exec_prefix=/usr
# set CMAKE configfile for target
export CMAKE_CONF=$ROOT/$TOOLCHAIN/etc/cmake-$TARGET_NAME.conf

View File

@ -31,7 +31,7 @@ PKG_SHORTDESC="python: The Python programming language"
PKG_LONGDESC="Python is an interpreted object-oriented programming language, and is often compared with Tcl, Perl, Java or Scheme."
PKG_IS_ADDON="no"
PKG_AUTORECONF="no"
PKG_AUTORECONF="yes"
PY_DISABLED_MODULES="readline _curses _curses_panel _tkinter nis gdbm bsddb ossaudiodev"
@ -64,16 +64,21 @@ PKG_CONFIGURE_OPTS_TARGET="ac_cv_file_dev_ptc=no \
--without-cxx-main \
--with-system-ffi \
--with-system-expat"
post_patch() {
# This is needed to make sure the Python build process doesn't try to
# regenerate those files with the pgen program. Otherwise, it builds
# pgen for the target, and tries to run it on the host.
touch $PKG_BUILD/Include/graminit.h
touch $PKG_BUILD/Python/graminit.c
}
make_host() {
make PYTHON_MODULES_INCLUDE="$HOST_INCDIR" \
PYTHON_MODULES_LIB="$HOST_LIBDIR" \
PYTHON_DISABLE_MODULES="$PY_DISABLED_MODULES"
sed -e "s|$ROOT/$TOOLCHAIN/include|$SYSROOT_PREFIX/usr/include|g" \
-e "s|$ROOT/$TOOLCHAIN/lib|$SYSROOT_PREFIX/usr/lib|g" \
-e "s|$ROOT/$TOOLCHAIN/bin/host-gcc|${TARGET_PREFIX}gcc|g" \
-i build/lib.linux-$(uname -m)-2.7/_sysconfigdata.py
# python distutils per default adds -L$LIBDIR when linking binary extensions
sed -e "s|^ 'LIBDIR':.*| 'LIBDIR': '/usr/lib',|g" -i $(cat pybuilddir.txt)/_sysconfigdata.py
}
makeinstall_host() {
@ -81,28 +86,14 @@ makeinstall_host() {
PYTHON_MODULES_LIB="$HOST_LIBDIR" \
PYTHON_DISABLE_MODULES="$PY_DISABLED_MODULES" \
install
cp Parser/pgen $ROOT/$TOOLCHAIN/bin
# replace python-config to make sure python uses $SYSROOT_PREFIX
mkdir -p $ROOT/$TOOLCHAIN/bin
rm -rf $ROOT/$TOOLCHAIN/bin/python*-config
sed -e "s:%PREFIX%:$SYSROOT_PREFIX/usr:g" -e "s:%CFLAGS%:$TARGET_CFLAGS:g" \
$PKG_DIR/scripts/python-config > $ROOT/$TOOLCHAIN/bin/python2.7-config
chmod +x $ROOT/$TOOLCHAIN/bin/python2.7-config
ln -s python2.7-config $ROOT/$TOOLCHAIN/bin/python-config
}
pre_configure_target() {
export PYTHON_FOR_BUILD=$ROOT/$TOOLCHAIN/bin/python
export BLDSHARED="$CC -shared"
export RUNSHARED="LD_LIBRARY_PATH=$ROOT/$TOOLCHAIN/lib:$LD_LIBRARY_PATH"
}
make_target() {
make -j1 CC="$TARGET_CC" \
HOSTPGEN=$ROOT/$TOOLCHAIN/bin/pgen \
make -j1 CC="$TARGET_CC" LDFLAGS="$TARGET_LDFLAGS -L." \
PYTHON_DISABLE_MODULES="$PY_DISABLED_MODULES" \
PYTHON_MODULES_INCLUDE="$TARGET_INCDIR" \
PYTHON_MODULES_LIB="$TARGET_LIBDIR"
@ -111,19 +102,13 @@ make_target() {
makeinstall_target() {
make -j1 CC="$TARGET_CC" \
DESTDIR=$SYSROOT_PREFIX \
HOSTPGEN=$ROOT/$TOOLCHAIN/bin/pgen \
PYTHON_DISABLE_MODULES="$PY_DISABLED_MODULES" \
PYTHON_MODULES_INCLUDE="$TARGET_INCDIR" \
PYTHON_MODULES_LIB="$TARGET_LIBDIR" \
install
# python distutils per default adds -L$LIBDIR when linking binary extensions
sed -e "s|^LIBDIR=.*|LIBDIR= $SYSROOT_PREFIX/usr/lib|" \
-i $SYSROOT_PREFIX/usr/lib/python*/config/Makefile
make -j1 CC="$TARGET_CC" \
DESTDIR=$INSTALL \
HOSTPGEN=$ROOT/$TOOLCHAIN/bin/pgen \
PYTHON_DISABLE_MODULES="$PY_DISABLED_MODULES" \
PYTHON_MODULES_INCLUDE="$TARGET_INCDIR" \
PYTHON_MODULES_LIB="$TARGET_LIBDIR" \
@ -139,13 +124,6 @@ post_makeinstall_target() {
python -Wi -t -B ../Lib/compileall.py $INSTALL/usr/lib/python*/ -f
rm -rf `find $INSTALL/usr/lib/python*/ -name "*.py"`
if [ ! -f $INSTALL/usr/lib/python*/lib-dynload/_socket.so ]; then
echo "sometimes Python dont build '_socket.so' for some reasons and continues without failing,"
echo "let it fail here, to be sure '_socket.so' will be installed. A rebuild of Python fixes"
echo "the issue in most cases"
exit 1
fi
rm -rf $INSTALL/usr/lib/python*/config
rm -rf $INSTALL/usr/bin/2to3
rm -rf $INSTALL/usr/bin/idle

View File

@ -2,22 +2,6 @@ diff --git a/Makefile.pre.in b/Makefile.pre.in
index bcd83bf..5e54a0d 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -191,6 +191,7 @@ UNICODE_OBJS= @UNICODE_OBJS@
PYTHON= python$(EXE)
BUILDPYTHON= python$(BUILDEXE)
+HOSTPYTHON= ./$(BUILDPYTHON)
PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
_PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
@@ -231,6 +232,7 @@ LIBFFI_INCLUDEDIR= @LIBFFI_INCLUDEDIR@
##########################################################################
# Parser
PGEN= Parser/pgen$(EXE)
+HOSTPGEN= $(PGEN)
PSRCS= \
Parser/acceler.c \
@@ -461,6 +463,7 @@ sharedmods: $(BUILDPYTHON) pybuilddir.txt
esac; \
$(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' \
@ -26,15 +10,6 @@ index bcd83bf..5e54a0d 100644
$(PYTHON_FOR_BUILD) $(srcdir)/setup.py $$quiet build
# Build static library
@@ -598,7 +601,7 @@ Modules/pwdmodule.o: $(srcdir)/Modules/pwdmodule.c $(srcdir)/Modules/posixmodule
$(GRAMMAR_H): $(GRAMMAR_INPUT) $(PGENSRCS)
@$(MKDIR_P) Include
$(MAKE) $(PGEN)
- $(PGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C)
+ $(HOSTPGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C)
$(GRAMMAR_C): $(GRAMMAR_H) $(GRAMMAR_INPUT) $(PGENSRCS)
$(MAKE) $(GRAMMAR_H)
touch $(GRAMMAR_C)
@@ -1142,7 +1145,9 @@ libainstall: all python-config
# Install the dynamically loadable modules
# This goes into $(exec_prefix)

View File

@ -0,0 +1,54 @@
Adjust library/header paths for cross-compilation
When cross-compiling third-party extensions, the get_python_inc() or
get_python_lib() can be called, to return the path to headers or
libraries. However, they use the sys.prefix of the host Python, which
returns incorrect paths when cross-compiling (paths pointing to host
headers and libraries).
In order to fix this, we introduce the _python_sysroot, _python_prefix
and _python_exec_prefix variables, that allow to override these
values, and get correct header/library paths when cross-compiling
third-party Python modules.
The _python_sysroot variable is also used to prefix the LIBDIR value
taken from the sysconfigdata module.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Index: b/Lib/distutils/sysconfig.py
===================================================================
--- a/Lib/distutils/sysconfig.py
+++ b/Lib/distutils/sysconfig.py
@@ -19,8 +19,13 @@
from distutils.errors import DistutilsPlatformError
# These are needed in a couple of spots, so just compute them once.
-PREFIX = os.path.normpath(sys.prefix)
-EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
+if "_python_sysroot" in os.environ:
+ _sysroot=os.environ.get('_python_sysroot')
+ PREFIX = os.path.normpath(_sysroot + os.environ.get('_python_prefix'))
+ EXEC_PREFIX = os.path.normpath(_sysroot + os.environ.get('_python_exec_prefix'))
+else:
+ PREFIX = os.path.normpath(sys.prefix)
+ EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
# Path to the base directory of the project. On Windows the binary may
# live in project/PCBuild9. If we're dealing with an x64 Windows build,
Index: b/Lib/distutils/command/build_ext.py
===================================================================
--- a/Lib/distutils/command/build_ext.py
+++ b/Lib/distutils/command/build_ext.py
@@ -237,7 +237,10 @@
if (sysconfig.get_config_var('Py_ENABLE_SHARED')):
if not sysconfig.python_build:
# building third party extensions
- self.library_dirs.append(sysconfig.get_config_var('LIBDIR'))
+ libdir = sysconfig.get_config_var('LIBDIR')
+ if "_python_sysroot" in os.environ:
+ libdir = os.environ.get("_python_sysroot") + libdir
+ self.library_dirs.append(libdir)
else:
# building python standard extensions
self.library_dirs.append('.')

View File

@ -0,0 +1,208 @@
Add a backport of http://bugs.python.org/issue16235 so we can use
python-config for cross builds.
This basically replaces the python version of python-config with a pure-shell
version that's already preprocessed when installed and doesn't depend
on the sysconfig import that usually leads to bad data/results.
Signed-off-by: Gustavo Zacarias <gustavo@zacarias.com.ar>
Index: b/configure.ac
===================================================================
--- a/configure.ac
+++ b/configure.ac
@@ -885,6 +885,7 @@
# Other platforms follow
if test $enable_shared = "yes"; then
+ PY_ENABLE_SHARED=1
AC_DEFINE(Py_ENABLE_SHARED, 1, [Defined if Python is built as a shared library.])
case $ac_sys_system in
BeOS*)
@@ -945,6 +946,7 @@
esac
else # shared is disabled
+ PY_ENABLE_SHARED=0
case $ac_sys_system in
CYGWIN*)
BLDLIBRARY='$(LIBRARY)'
@@ -1921,6 +1923,9 @@
AC_SUBST(BLDSHARED)
AC_SUBST(CCSHARED)
AC_SUBST(LINKFORSHARED)
+AC_SUBST(PY_ENABLE_SHARED)
+LIBPL="${prefix}/lib/python${VERSION}/config"
+AC_SUBST(LIBPL)
# SO is the extension of shared libraries `(including the dot!)
# -- usually .so, .sl on HP-UX, .dll on Cygwin
AC_MSG_CHECKING(SO)
@@ -4588,7 +4593,7 @@
AC_SUBST(ENSUREPIP)
# generate output files
-AC_CONFIG_FILES(Makefile.pre Modules/Setup.config Misc/python.pc)
+AC_CONFIG_FILES(Makefile.pre Modules/Setup.config Misc/python.pc Misc/python-config.sh)
AC_CONFIG_FILES([Modules/ld_so_aix], [chmod +x Modules/ld_so_aix])
AC_OUTPUT
Index: b/Makefile.pre.in
===================================================================
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -166,7 +166,7 @@
SUBDIRSTOO= Include Lib Misc Demo
# Files and directories to be distributed
-CONFIGFILES= configure configure.ac acconfig.h pyconfig.h.in Makefile.pre.in
+CONFIGFILES= configure configure.ac acconfig.h pyconfig.h.in Makefile.pre.in Misc/python-config.sh
DISTFILES= README ChangeLog $(CONFIGFILES)
DISTDIRS= $(SUBDIRS) $(SUBDIRSTOO) Ext-dummy
DIST= $(DISTFILES) $(DISTDIRS)
@@ -410,7 +410,7 @@
# Default target
all: build_all
-build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks
+build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks python-config
# Compile a binary with gcc profile guided optimization.
profile-opt:
@@ -1101,10 +1101,12 @@
fi; \
cd $(srcdir)/Lib/$(PLATDIR); $(RUNSHARED) ./regen
-python-config: $(srcdir)/Misc/python-config.in
+python-config: $(srcdir)/Misc/python-config.in Misc/python-config.sh
# Substitution happens here, as the completely-expanded BINDIR
# is not available in configure
- sed -e "s,@EXENAME@,$(BINDIR)/python$(VERSION)$(EXE)," < $(srcdir)/Misc/python-config.in >python-config
+ sed -e "s,@EXENAME@,$(BINDIR)/python$(VERSION)$(EXE)," < $(srcdir)/Misc/python-config.in >python-config.py
+ # Replace makefile compat. variable references with shell script compat. ones; $(VAR) -> ${VAR}
+ sed -e 's,\$$(\([A-Za-z0-9_]*\)),\$$\{\1\},g' Misc/python-config.sh >python-config
# Install the include files
INCLDIRSTOMAKE=$(INCLUDEDIR) $(CONFINCLUDEDIR) $(INCLUDEPY) $(CONFINCLUDEPY)
@@ -1163,7 +1165,7 @@
$(INSTALL_SCRIPT) $(srcdir)/Modules/makesetup $(DESTDIR)$(LIBPL)/makesetup
$(INSTALL_SCRIPT) $(srcdir)/install-sh $(DESTDIR)$(LIBPL)/install-sh
$(INSTALL_SCRIPT) python-config $(DESTDIR)$(BINDIR)/python$(VERSION)-config
- rm python-config
+ $(INSTALL_SCRIPT) python-config.py $(DESTDIR)$(LIBPL)/python-config.py
@if [ -s Modules/python.exp -a \
"`echo $(MACHDEP) | sed 's/^\(...\).*/\1/'`" = "aix" ]; then \
echo; echo "Installing support files for building shared extension modules on AIX:"; \
@@ -1345,6 +1347,7 @@
config.cache config.log pyconfig.h Modules/config.c
-rm -rf build platform
-rm -rf $(PYTHONFRAMEWORKDIR)
+ -rm -f python-config.py python-config
# Make things extra clean, before making a distribution:
# remove all generated files, even Makefile[.pre]
Index: b/Misc/python-config.sh.in
===================================================================
--- /dev/null
+++ b/Misc/python-config.sh.in
@@ -0,0 +1,102 @@
+#!/bin/sh
+
+exit_with_usage ()
+{
+ echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--configdir"
+ exit $1
+}
+
+if [ "$1" = "" ] ; then
+ exit_with_usage 1
+fi
+
+# Returns the actual prefix where this script was installed to.
+installed_prefix ()
+{
+ RESULT=$(dirname $(cd $(dirname "$1") && pwd -P))
+ if which readlink >/dev/null 2>&1 ; then
+ RESULT=$(readlink -f "$RESULT")
+ fi
+ echo $RESULT
+}
+
+prefix_build="@prefix@"
+prefix_real=$(installed_prefix "$0")
+
+# Use sed to fix paths from their built to locations to their installed to locations.
+prefix=$(echo "$prefix_build" | sed "s#$prefix_build#$prefix_real#")
+exec_prefix_build="@exec_prefix@"
+exec_prefix=$(echo "$exec_prefix_build" | sed "s#$exec_prefix_build#$prefix_real#")
+includedir=$(echo "@includedir@")
+libdir=$(echo "@libdir@")
+CFLAGS=$(echo "@CFLAGS@" | sed "s#$prefix_build#$prefix_real#")
+VERSION="@VERSION@"
+LIBM="@LIBM@"
+LIBC="@LIBC@"
+SYSLIBS="$LIBM $LIBC"
+LIBS="@LIBS@ $SYSLIBS -lpython${VERSION}"
+BASECFLAGS="@BASECFLAGS@"
+LDLIBRARY="@LDLIBRARY@"
+LINKFORSHARED="@LINKFORSHARED@"
+OPT="@OPT@"
+PY_ENABLE_SHARED="@PY_ENABLE_SHARED@"
+LDVERSION="@LDVERSION@"
+LIBDEST=${prefix}/lib/python${VERSION}
+LIBPL=$(echo "@LIBPL@" | sed "s#$prefix_build#$prefix_real#")
+SO="@SO@"
+PYTHONFRAMEWORK="@PYTHONFRAMEWORK@"
+INCDIR="-I$includedir/python${VERSION}"
+PLATINCDIR="-I$includedir/python${VERSION}"
+
+# Scan for --help or unknown argument.
+for ARG in $*
+do
+ case $ARG in
+ --help)
+ exit_with_usage 0
+ ;;
+ --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--abiflags|--configdir)
+ ;;
+ *)
+ exit_with_usage 1
+ ;;
+esac
+done
+
+for ARG in "$@"
+do
+ case "$ARG" in
+ --prefix)
+ echo "$prefix"
+ ;;
+ --exec-prefix)
+ echo "$exec_prefix"
+ ;;
+ --includes)
+ echo "$INCDIR $PLATINCDIR"
+ ;;
+ --cflags)
+ echo "$INCDIR $PLATINCDIR $BASECFLAGS $CFLAGS $OPT"
+ ;;
+ --libs)
+ echo "$LIBS"
+ ;;
+ --ldflags)
+ LINKFORSHAREDUSED=
+ if [ -z "$PYTHONFRAMEWORK" ] ; then
+ LINKFORSHAREDUSED=$LINKFORSHARED
+ fi
+ LIBPLUSED=
+ if [ "$PY_ENABLE_SHARED" = "0" ] ; then
+ LIBPLUSED="-L$LIBPL"
+ fi
+ echo "$LIBPLUSED -L$libdir $LIBS $LINKFORSHAREDUSED"
+ ;;
+ --extension-suffix)
+ echo "$SO"
+ ;;
+ --configdir)
+ echo "$LIBPL"
+ ;;
+esac
+done

View File

@ -0,0 +1,21 @@
Abort on failed module build
When building a Python module fails, the setup.py script currently
doesn't exit with an error, and simply continues. This is not a really
nice behavior, so this patch changes setup.py to abort with an error,
so that the build issue is clearly noticeable.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Index: b/setup.py
===================================================================
--- a/setup.py
+++ b/setup.py
@@ -283,6 +283,7 @@
print "Failed to build these modules:"
print_three_column(failed)
print
+ sys.exit(1)
def build_extension(self, ext):

View File

@ -1,31 +0,0 @@
#!/bin/sh
################################################################################
# This file is part of OpenELEC - http://www.openelec.tv
# Copyright (C) 2009-2014 Stephan Raue (stephan@openelec.tv)
#
# OpenELEC is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# OpenELEC is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenELEC. If not, see <http://www.gnu.org/licenses/>.
################################################################################
case $1 in
--prefix) echo "/usr" ;;
--exec-prefix) echo "/usr" ;;
--includes) echo "-I%PREFIX%/include/python2.7" ;;
--libs) echo "-lpthread -ldl -lutil -lm -lpython2.7" ;;
--cflags) echo "-I%PREFIX%/include/python2.7 -DNDEBUG %CFLAGS%" ;;
--ldflags) echo "-lpthread -ldl -lutil -lm -lpython2.7" ;;
esac
exit 0

View File

@ -34,10 +34,14 @@ PKG_AUTORECONF="yes"
PKG_CONFIGURE_OPTS_TARGET="--enable-thread --disable-introspection"
pre_configure_target() {
export PYTHON_INCLUDES="$($SYSROOT_PREFIX/usr/bin/python2-config --includes)"
}
post_makeinstall_target() {
find $INSTALL/usr/lib -name "*.py" -exec rm -rf "{}" ";"
find $INSTALL/usr/lib -name "*.pyc" -exec rm -rf "{}" ";"
rm -rf $INSTALL/usr/bin
rm -rf $INSTALL/usr/share/pygobject
}
}

View File

@ -0,0 +1,32 @@
[PATCH] m4/python.m4: add PYTHON_INCLUDES override for cross compilation
As the configure script mixes up host/target python. Equivalent to the
similar code in dbus-python.
Signed-off-by: Peter Korsgaard <peter@korsgaard.com>
---
m4/python.m4 | 4 ++++
1 file changed, 4 insertions(+)
Index: python-gobject-2.28.6/m4/python.m4
===================================================================
--- python-gobject-2.28.6.orig/m4/python.m4
+++ python-gobject-2.28.6/m4/python.m4
@@ -43,6 +43,9 @@
[AC_REQUIRE([AM_PATH_PYTHON])
AC_MSG_CHECKING(for headers required to compile python extensions)
dnl deduce PYTHON_INCLUDES
+if test "${PYTHON_INCLUDES+set}" = set; then
+ AC_MSG_NOTICE([PYTHON_INCLUDES overridden to: $PYTHON_INCLUDES])
+else
py_prefix=`$PYTHON -c "import sys; sys.stdout.write(sys.prefix)"`
py_exec_prefix=`$PYTHON -c "import sys; sys.stdout.write(sys.exec_prefix)"`
PYTHON_CONFIG=`which $PYTHON`-config
@@ -54,6 +57,7 @@
PYTHON_INCLUDES="$PYTHON_INCLUDES -I${py_exec_prefix}/include/python${PYTHON_VERSION}"
fi
fi
+fi
AC_SUBST(PYTHON_INCLUDES)
dnl check if the headers exist:
save_CPPFLAGS="$CPPFLAGS"

View File

@ -32,6 +32,11 @@ PKG_LONGDESC="D-BUS is a message bus, used for sending messages between applicat
PKG_IS_ADDON="no"
PKG_AUTORECONF="yes"
pre_configure_target() {
export PYTHON_INCLUDES="$($SYSROOT_PREFIX/usr/bin/python2-config --includes)"
export PYTHON_LIBS="$($SYSROOT_PREFIX/usr/bin/python2-config --ldflags)"
}
post_makeinstall_target() {
find $INSTALL/usr/lib -name "*.py" -exec rm -rf "{}" ";"
find $INSTALL/usr/lib -name "*.pyc" -exec rm -rf "{}" ";"