tools/download-cleaner: add new tool

This script selectively deletes files out of the specified download
directory to prune source tarballs that don't correspond with the
current version of its package in the tree according to the set
PROJECT, DEVICE and ARCH.

The script is also capable of building a new SOURCES dir to combine
different combinations of PROJECT, DEVICE and ARCH tarballs. This is
to move out known good tarballs to a new directory, and to delete the
old SOURCES dir once all desired tarballs have been moved.

Signed-off-by: Ian Leonard <antonlacon@gmail.com>

tools/download-cleaner

@@ -0,0 +1,137 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2018-2020 Ian Leonard (antonlacon@gmail.com)
+#
+# Scan through the given directory looking for out of date source packages.
+# If a source tarball is found that is different from what is currently in
+# tree, then print out the discovered files.
+#
+# If -d is passed, also delete obsolete files.
+# If -f is passed, fetch updated source tarballs.
+# If -b is passed, build a new SOURCES, populated from the old SOURCES
+#   Set NEW_SOURCES_DIR to customize the temp dir to use in building
+# If -r is passed, replace the old SOURCES when finished building new SOURCES
+
+set -e
+
+. config/options ""
+
+# default variables
+DESTRUCTIVE_RUN="false"
+FETCH_UPDATES="false"
+PACKAGES_TO_UPDATE=()
+BUILD_SOURCES="false"
+REPLACE_OLD="false"
+NEW_SOURCES="${NEW_SOURCES_DIR:-${SOURCES}.new}"
+
+
+# helper functions
+help(){
+  echo "Usage: ${0} [-hdfbr]"
+    echo "Set PROJECT, DEVICE and ARCH as required."
+    echo "  -h this help"
+    echo "  -d delete obsolete source packages (default no)"
+    echo "  -f fetch updated source packages (default: no)"
+    echo "  -b build new SOURCES content (default: no)"
+    echo "  -r replace old SOURCES when rebuilding (default: no)"
+}
+
+
+# command line opts
+while getopts hdfrb OPT; do
+  case "${OPT}" in
+    h)
+      help
+      exit 0
+      ;;
+    d)
+      DESTRUCTIVE_RUN="true"
+      ;;
+    f)
+      FETCH_UPDATES="true"
+      ;;
+    b)
+      BUILD_SOURCES="true"
+      ;;
+    r)
+      REPLACE_OLD="true"
+      ;;
+    \?)
+      # error and output help on unknown
+      help
+      die
+      ;;
+  esac
+done
+
+shift $((${OPTIND}-1))
+
+
+# sanity checking
+if [ ! -d "${SOURCES}" ]; then
+  die "error: ${SOURCES} is not a directory"
+elif [ "${DESTRUCTIVE_RUN}" = "true" -a "${BUILD_SOURCES}" = "true" ]; then
+  die "error: options '-d' and '-b' are mutually exclusive"
+fi
+
+
+# main
+# process files in SOURCES
+for SOURCE_PACKAGE in $(find "${SOURCES}/" -mindepth 1 -type d); do
+  PACKAGE_NAME=$(basename "${SOURCE_PACKAGE}")
+
+  # check if package is still in the tree to selectively prune, or delete the dir
+  PACKAGE_DIR=$(get_pkg_directory "${PACKAGE_NAME}" || true)
+  if [ -n "${PACKAGE_DIR}" ]; then
+    PACKAGE_SOURCE_FILE=$(get_pkg_variable "${PACKAGE_NAME}" "PKG_SOURCE_NAME")
+  fi
+  # PACKAGE_DIR is null if not in tree, PACKAGE_SOURCE_FILE is null if PKG_ARCH mismatch
+  if [ -n "${PACKAGE_DIR}" -a -n "${PACKAGE_SOURCE_FILE}" ]; then
+    CUR_PACKAGE_FILE=$(basename "${PACKAGE_SOURCE_FILE}")
+
+    for FILE in $(find "${SOURCE_PACKAGE}/" -type f); do
+      # don't test auxilliary files
+      if [[ "${FILE}" = *.url ]] || [[ "${FILE}" = *.sha256 ]]; then
+        continue
+      fi
+
+      # obsolete file handling
+      if [ "${FILE}" != "${SOURCES}/${PACKAGE_NAME}/${CUR_PACKAGE_FILE}" ]; then
+        ls -1 "${FILE}"*
+        PACKAGES_TO_UPDATE+="${PACKAGE_NAME}"
+
+        if [ "${DESTRUCTIVE_RUN}" = "true" ]; then
+          rm "${FILE}"*
+        fi
+      # current file handling
+      elif [ "${BUILD_SOURCES}" = "true" ]; then
+        echo "Relocating ${PACKAGE_NAME} files..."
+        mkdir -p "${NEW_SOURCES}/${PACKAGE_NAME}"
+        mv "${FILE}"* "${NEW_SOURCES}/${PACKAGE_NAME}/"
+      fi
+    done
+  else
+    echo "info: ${PACKAGE_NAME} no longer in tree"
+    if [ "${DESTRUCTIVE_RUN}" = "true" ]; then
+      rm -r "${SOURCE_PACKAGE}"
+    fi
+  fi
+done
+
+if [ "${BUILD_SOURCES}" = "true" -a "${REPLACE_OLD}" = "true" ]; then
+  rm -r "${SOURCES}"
+  mv "${NEW_SOURCES}" "${SOURCES}"
+fi
+
+if [ "${FETCH_UPDATES}" = "true" ]; then
+  if "${BUILD_SOURCES}" = "true" -a "${REPLACE_OLD}" = "false" ]; then
+    SOURCES="${NEW_SOURCES}"
+  fi
+  for PACKAGE in "${PACKAGES_TO_UPDATE[@]}"; do
+    # scripts/get determines if tarball is present before downloading
+    SOURCES_DIR="${SOURCES}" "${SCRIPTS}/get" "${PACKAGE}"
+  done
+fi
+
+exit