diff --git a/packages/compress/zstd/package.mk b/packages/compress/zstd/package.mk new file mode 100644 index 0000000000..0266d60b71 --- /dev/null +++ b/packages/compress/zstd/package.mk @@ -0,0 +1,34 @@ +################################################################################ +# This file is part of LibreELEC - https://libreelec.tv +# Copyright (C) 2017-present Team LibreELEC +# +# LibreELEC is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# LibreELEC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with LibreELEC. If not, see . +################################################################################ + +PKG_NAME="zstd" +PKG_VERSION="1.3.1" +PKG_SHA256="312fb9dc75668addbc9c8f33c7fa198b0fc965c576386b8451397e06256eadc6" +PKG_ARCH="any" +PKG_LICENSE="BSD/GPLv2" +PKG_SITE="http://www.zstd.net" +PKG_URL="https://github.com/facebook/zstd/archive/v${PKG_VERSION}.tar.gz" +PKG_SOURCE_DIR=$PKG_NAME-$PKG_VERSION +PKG_DEPENDS_TARGET="toolchain" +PKG_SECTION="compress" +PKG_SHORTDESC="fast real-time compression algorithm" + +PKG_AUTORECONF="no" + +PKG_CMAKE_SCRIPT="$PKG_BUILD/build/cmake/CMakeLists.txt" +PKG_CMAKE_OPTS_HOST="-DTHREADS_PTHREAD_ARG=0" diff --git a/packages/sysutils/squashfs/package.mk b/packages/sysutils/squashfs/package.mk index 2f1caf7275..222cd0c01f 100644 --- a/packages/sysutils/squashfs/package.mk +++ b/packages/sysutils/squashfs/package.mk @@ -1,30 +1,30 @@ ################################################################################ -# This file is part of OpenELEC - http://www.openelec.tv -# Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# This file is part of LibreELEC - https://libreelec.tv +# Copyright (C) 2017-present Team LibreELEC # -# OpenELEC is free software: you can redistribute it and/or modify +# LibreELEC is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 2 of the License, or # (at your option) any later version. # -# OpenELEC is distributed in the hope that it will be useful, +# LibreELEC is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with OpenELEC. If not, see . +# along with LibreELEC. If not, see . ################################################################################ PKG_NAME="squashfs" -PKG_VERSION="4.3" -PKG_SHA256="0d605512437b1eb800b4736791559295ee5f60177e102e4d4ccd0ee241a5f3f6" +PKG_VERSION="1a6ffc7" +PKG_SHA256="2a641ae2f3ae772b9b34dce955a77cfa0717eda57efb3df3a4a5222607c24b2c" PKG_ARCH="any" PKG_LICENSE="GPL" -PKG_SITE="http://squashfs.sourceforge.net/" -PKG_URL="$SOURCEFORGE_SRC/squashfs/squashfs/${PKG_NAME}${PKG_VERSION}/${PKG_NAME}${PKG_VERSION}.tar.gz" -PKG_SOURCE_DIR="${PKG_NAME}${PKG_VERSION}" -PKG_DEPENDS_HOST="ccache:host zlib:host lzo:host xz:host" +PKG_SITE="https://git.kernel.org/pub/scm/fs/squashfs/squashfs-tools.git" +PKG_URL="https://git.kernel.org/pub/scm/fs/squashfs/squashfs-tools.git/snapshot/$PKG_VERSION.tar.gz" +PKG_SOURCE_DIR="$PKG_VERSION" +PKG_DEPENDS_HOST="ccache:host zlib:host lzo:host xz:host zstd:host" PKG_SECTION="sysutils" PKG_SHORTDESC="squashfs-tools: A compressed read-only filesystem for Linux" PKG_LONGDESC="Squashfs is intended to be a general read-only filesystem, for archival use (i.e. in cases where a .tar.gz file may be used), and in constrained block device/memory systems (e.g. embedded systems) where low overhead is needed. The filesystem is currently stable and has been tested on PowerPC, i386, SPARC and ARM architectures." @@ -34,7 +34,7 @@ PKG_AUTORECONF="no" make_host() { make -C squashfs-tools mksquashfs \ - XZ_SUPPORT=1 LZO_SUPPORT=1 \ + XZ_SUPPORT=1 LZO_SUPPORT=1 ZSTD_SUPPORT=1 \ XATTR_SUPPORT=0 XATTR_DEFAULT=0 \ INCLUDEDIR="-I. -I$TOOLCHAIN/include" } diff --git a/packages/sysutils/squashfs/patches/squashfs-001-add-zstd-support.patch b/packages/sysutils/squashfs/patches/squashfs-001-add-zstd-support.patch new file mode 100644 index 0000000000..b38930fdc4 --- /dev/null +++ b/packages/sysutils/squashfs/patches/squashfs-001-add-zstd-support.patch @@ -0,0 +1,418 @@ +From cc08b43a31fed1289c2027d5090999da569457f1 Mon Sep 17 00:00:00 2001 +From: Sean Purcell +Date: Thu, 3 Aug 2017 17:47:03 -0700 +Subject: [PATCH v5] squashfs-tools: Add zstd support + +This patch adds zstd support to squashfs-tools. It works with zstd +versions >= 1.0.0. It was originally written by Sean Purcell. + +Signed-off-by: Sean Purcell +Signed-off-by: Nick Terrell +--- +v4 -> v5: +- Fix patch documentation to reflect that Sean Purcell is the author +- Don't strip trailing whitespace of unreleated code +- Make zstd_display_options() static + + squashfs-tools/Makefile | 21 ++++ + squashfs-tools/compressor.c | 8 ++ + squashfs-tools/squashfs_fs.h | 1 + + squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++++++++++ + squashfs-tools/zstd_wrapper.h | 48 ++++++++ + 5 files changed, 332 insertions(+) + create mode 100644 squashfs-tools/zstd_wrapper.c + create mode 100644 squashfs-tools/zstd_wrapper.h + +diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile +index 52d2582..8e82e09 100644 +--- a/squashfs-tools/Makefile ++++ b/squashfs-tools/Makefile +@@ -75,6 +75,19 @@ GZIP_SUPPORT = 1 + #LZMA_SUPPORT = 1 + #LZMA_DIR = ../../../../LZMA/lzma465 + ++ ++########### Building ZSTD support ############ ++# ++# The ZSTD library is supported ++# ZSTD homepage: http://zstd.net ++# ZSTD source repository: https://github.com/facebook/zstd ++# ++# To build configure the tools using cmake to build shared libraries, ++# install and uncomment ++# the ZSTD_SUPPORT line below. ++# ++#ZSTD_SUPPORT = 1 ++ + ######## Specifying default compression ######## + # + # The next line specifies which compression algorithm is used by default +@@ -177,6 +190,14 @@ LIBS += -llz4 + COMPRESSORS += lz4 + endif + ++ifeq ($(ZSTD_SUPPORT),1) ++CFLAGS += -DZSTD_SUPPORT ++MKSQUASHFS_OBJS += zstd_wrapper.o ++UNSQUASHFS_OBJS += zstd_wrapper.o ++LIBS += -lzstd ++COMPRESSORS += zstd ++endif ++ + ifeq ($(XATTR_SUPPORT),1) + ifeq ($(XATTR_DEFAULT),1) + CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT +diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c +index 525e316..02b5e90 100644 +--- a/squashfs-tools/compressor.c ++++ b/squashfs-tools/compressor.c +@@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = { + extern struct compressor xz_comp_ops; + #endif + ++#ifndef ZSTD_SUPPORT ++static struct compressor zstd_comp_ops = { ++ ZSTD_COMPRESSION, "zstd" ++}; ++#else ++extern struct compressor zstd_comp_ops; ++#endif + + static struct compressor unknown_comp_ops = { + 0, "unknown" +@@ -77,6 +84,7 @@ struct compressor *compressor[] = { + &lzo_comp_ops, + &lz4_comp_ops, + &xz_comp_ops, ++ &zstd_comp_ops, + &unknown_comp_ops + }; + +diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h +index 791fe12..afca918 100644 +--- a/squashfs-tools/squashfs_fs.h ++++ b/squashfs-tools/squashfs_fs.h +@@ -277,6 +277,7 @@ typedef long long squashfs_inode; + #define LZO_COMPRESSION 3 + #define XZ_COMPRESSION 4 + #define LZ4_COMPRESSION 5 ++#define ZSTD_COMPRESSION 6 + + struct squashfs_super_block { + unsigned int s_magic; +diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c +new file mode 100644 +index 0000000..dcab75a +--- /dev/null ++++ b/squashfs-tools/zstd_wrapper.c +@@ -0,0 +1,254 @@ ++/* ++ * Copyright (c) 2017 ++ * Phillip Lougher ++ * ++ * This program is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License ++ * as published by the Free Software Foundation; either version 2, ++ * or (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * zstd_wrapper.c ++ * ++ * Support for ZSTD compression http://zstd.net ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "squashfs_fs.h" ++#include "zstd_wrapper.h" ++#include "compressor.h" ++ ++static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL; ++ ++/* ++ * This function is called by the options parsing code in mksquashfs.c ++ * to parse any -X compressor option. ++ * ++ * This function returns: ++ * >=0 (number of additional args parsed) on success ++ * -1 if the option was unrecognised, or ++ * -2 if the option was recognised, but otherwise bad in ++ * some way (e.g. invalid parameter) ++ * ++ * Note: this function sets internal compressor state, but does not ++ * pass back the results of the parsing other than success/failure. ++ * The zstd_dump_options() function is called later to get the options in ++ * a format suitable for writing to the filesystem. ++ */ ++static int zstd_options(char *argv[], int argc) ++{ ++ if (strcmp(argv[0], "-Xcompression-level") == 0) { ++ if (argc < 2) { ++ fprintf(stderr, "zstd: -Xcompression-level missing " ++ "compression level\n"); ++ fprintf(stderr, "zstd: -Xcompression-level it should " ++ "be 1 <= n <= %d\n", ZSTD_maxCLevel()); ++ goto failed; ++ } ++ ++ compression_level = atoi(argv[1]); ++ if (compression_level < 1 || ++ compression_level > ZSTD_maxCLevel()) { ++ fprintf(stderr, "zstd: -Xcompression-level invalid, it " ++ "should be 1 <= n <= %d\n", ZSTD_maxCLevel()); ++ goto failed; ++ } ++ ++ return 1; ++ } ++ ++ return -1; ++failed: ++ return -2; ++} ++ ++/* ++ * This function is called by mksquashfs to dump the parsed ++ * compressor options in a format suitable for writing to the ++ * compressor options field in the filesystem (stored immediately ++ * after the superblock). ++ * ++ * This function returns a pointer to the compression options structure ++ * to be stored (and the size), or NULL if there are no compression ++ * options. ++ */ ++static void *zstd_dump_options(int block_size, int *size) ++{ ++ static struct zstd_comp_opts comp_opts; ++ ++ /* don't return anything if the options are all default */ ++ if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL) ++ return NULL; ++ ++ comp_opts.compression_level = compression_level; ++ ++ SQUASHFS_INSWAP_COMP_OPTS(&comp_opts); ++ ++ *size = sizeof(comp_opts); ++ return &comp_opts; ++} ++ ++/* ++ * This function is a helper specifically for the append mode of ++ * mksquashfs. Its purpose is to set the internal compressor state ++ * to the stored compressor options in the passed compressor options ++ * structure. ++ * ++ * In effect this function sets up the compressor options ++ * to the same state they were when the filesystem was originally ++ * generated, this is to ensure on appending, the compressor uses ++ * the same compression options that were used to generate the ++ * original filesystem. ++ * ++ * Note, even if there are no compressor options, this function is still ++ * called with an empty compressor structure (size == 0), to explicitly ++ * set the default options, this is to ensure any user supplied ++ * -X options on the appending mksquashfs command line are over-ridden. ++ * ++ * This function returns 0 on sucessful extraction of options, and -1 on error. ++ */ ++static int zstd_extract_options(int block_size, void *buffer, int size) ++{ ++ struct zstd_comp_opts *comp_opts = buffer; ++ ++ if (size == 0) { ++ /* Set default values */ ++ compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL; ++ return 0; ++ } ++ ++ /* we expect a comp_opts structure of sufficient size to be present */ ++ if (size < sizeof(*comp_opts)) ++ goto failed; ++ ++ SQUASHFS_INSWAP_COMP_OPTS(comp_opts); ++ ++ if (comp_opts->compression_level < 1 || ++ comp_opts->compression_level > ZSTD_maxCLevel()) { ++ fprintf(stderr, "zstd: bad compression level in compression " ++ "options structure\n"); ++ goto failed; ++ } ++ ++ compression_level = comp_opts->compression_level; ++ ++ return 0; ++ ++failed: ++ fprintf(stderr, "zstd: error reading stored compressor options from " ++ "filesystem!\n"); ++ ++ return -1; ++} ++ ++static void zstd_display_options(void *buffer, int size) ++{ ++ struct zstd_comp_opts *comp_opts = buffer; ++ ++ /* we expect a comp_opts structure of sufficient size to be present */ ++ if (size < sizeof(*comp_opts)) ++ goto failed; ++ ++ SQUASHFS_INSWAP_COMP_OPTS(comp_opts); ++ ++ if (comp_opts->compression_level < 1 || ++ comp_opts->compression_level > ZSTD_maxCLevel()) { ++ fprintf(stderr, "zstd: bad compression level in compression " ++ "options structure\n"); ++ goto failed; ++ } ++ ++ printf("\tcompression-level %d\n", comp_opts->compression_level); ++ ++ return; ++ ++failed: ++ fprintf(stderr, "zstd: error reading stored compressor options from " ++ "filesystem!\n"); ++} ++ ++/* ++ * This function is called by mksquashfs to initialise the ++ * compressor, before compress() is called. ++ * ++ * This function returns 0 on success, and -1 on error. ++ */ ++static int zstd_init(void **strm, int block_size, int datablock) ++{ ++ ZSTD_CCtx *cctx = ZSTD_createCCtx(); ++ ++ if (!cctx) { ++ fprintf(stderr, "zstd: failed to allocate compression " ++ "context!\n"); ++ return -1; ++ } ++ ++ *strm = cctx; ++ return 0; ++} ++ ++static int zstd_compress(void *strm, void *dest, void *src, int size, ++ int block_size, int *error) ++{ ++ const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size, ++ src, size, compression_level); ++ ++ if (ZSTD_isError(res)) { ++ /* FIXME: ++ * zstd does not expose stable error codes. The error enum may ++ * change between versions. Until upstream zstd stablizes the ++ * error codes, we have no way of knowing why the error occurs. ++ * zstd shouldn't fail to compress any input unless there isn't ++ * enough output space. We assume that is the cause and return ++ * the special error code for not enough output space. ++ */ ++ return 0; ++ } ++ ++ return (int)res; ++} ++ ++static int zstd_uncompress(void *dest, void *src, int size, int outsize, ++ int *error) ++{ ++ const size_t res = ZSTD_decompress(dest, outsize, src, size); ++ ++ if (ZSTD_isError(res)) { ++ fprintf(stderr, "\t%d %d\n", outsize, size); ++ ++ *error = (int)ZSTD_getErrorCode(res); ++ return -1; ++ } ++ ++ return (int)res; ++} ++ ++static void zstd_usage(void) ++{ ++ fprintf(stderr, "\t -Xcompression-level \n"); ++ fprintf(stderr, "\t\t should be 1 .. %d (default " ++ "%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL); ++} ++ ++struct compressor zstd_comp_ops = { ++ .init = zstd_init, ++ .compress = zstd_compress, ++ .uncompress = zstd_uncompress, ++ .options = zstd_options, ++ .dump_options = zstd_dump_options, ++ .extract_options = zstd_extract_options, ++ .display_options = zstd_display_options, ++ .usage = zstd_usage, ++ .id = ZSTD_COMPRESSION, ++ .name = "zstd", ++ .supported = 1 ++}; +diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h +new file mode 100644 +index 0000000..4fbef0a +--- /dev/null ++++ b/squashfs-tools/zstd_wrapper.h +@@ -0,0 +1,48 @@ ++#ifndef ZSTD_WRAPPER_H ++#define ZSTD_WRAPPER_H ++/* ++ * Squashfs ++ * ++ * Copyright (c) 2017 ++ * Phillip Lougher ++ * ++ * This program is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License ++ * as published by the Free Software Foundation; either version 2, ++ * or (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * zstd_wrapper.h ++ * ++ */ ++ ++#ifndef linux ++#define __BYTE_ORDER BYTE_ORDER ++#define __BIG_ENDIAN BIG_ENDIAN ++#define __LITTLE_ENDIAN LITTLE_ENDIAN ++#else ++#include ++#endif ++ ++#if __BYTE_ORDER == __BIG_ENDIAN ++extern unsigned int inswap_le16(unsigned short); ++extern unsigned int inswap_le32(unsigned int); ++ ++#define SQUASHFS_INSWAP_COMP_OPTS(s) { \ ++ (s)->compression_level = inswap_le32((s)->compression_level); \ ++} ++#else ++#define SQUASHFS_INSWAP_COMP_OPTS(s) ++#endif ++ ++/* Default compression */ ++#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15 ++ ++struct zstd_comp_opts { ++ int compression_level; ++}; ++#endif +-- +2.9.3 diff --git a/scripts/image b/scripts/image index 2d079a1611..e6a3109455 100755 --- a/scripts/image +++ b/scripts/image @@ -234,13 +234,9 @@ rm -rf $TARGET_IMG/$IMAGE_NAME.kernel cp -PR $BUILD/linux-$(kernel_version)/arch/$TARGET_KERNEL_ARCH/boot/$KERNEL_TARGET $TARGET_IMG/$IMAGE_NAME.kernel chmod 0644 $TARGET_IMG/$IMAGE_NAME.kernel -# create squashfs file -if [ -z "$SQUASHFS_COMPRESSION" ]; then - SQUASHFS_COMPRESSION="gzip" -fi - +# create squashfs file, default to gzip if no compression configured echo "rm -rf \"$TARGET_IMG/$IMAGE_NAME.system\"" >> $FAKEROOT_SCRIPT -echo "$TOOLCHAIN/bin/mksquashfs \"$BUILD/image/system\" \"$TARGET_IMG/$IMAGE_NAME.system\" -noappend -comp $SQUASHFS_COMPRESSION" >> $FAKEROOT_SCRIPT +echo "$TOOLCHAIN/bin/mksquashfs \"$BUILD/image/system\" \"$TARGET_IMG/$IMAGE_NAME.system\" -noappend -comp ${SQUASHFS_COMPRESSION:-gzip}" >> $FAKEROOT_SCRIPT # run fakeroot $TOOLCHAIN/bin/fakeroot -- $FAKEROOT_SCRIPT