build: use xargs to collect packages' json info
This commit is contained in:
Andre Heider
MilhouseVH
@ -3,46 +3,8 @@
|
||||
|
||||
THREADCOUNT=${THREADCOUNT:-$(nproc)}
|
||||
|
||||
# This function is passed a list of package.mk paths to be processed.
|
||||
# Each package.mk is sourced with relevant variables output in JSON format.
|
||||
json_worker() {
|
||||
local packages="$@"
|
||||
local pkgpath hierarchy exited
|
||||
|
||||
exit() { exited=1; }
|
||||
|
||||
. config/options ""
|
||||
|
||||
for pkgpath in ${packages}; do
|
||||
pkgpath="${pkgpath%%@*}"
|
||||
|
||||
exited=0
|
||||
if ! source_package "${pkgpath}/package.mk" &>/dev/null; then
|
||||
unset -f exit
|
||||
die "$(print_color CLR_ERROR "FAILURE: sourcing package ${pkgpath}/package.mk")"
|
||||
fi
|
||||
|
||||
[ ${exited} -eq 1 ] && continue
|
||||
|
||||
[[ ${pkgpath} =~ ^${ROOT}/${PACKAGES}/ ]] && hierarchy="global" || hierarchy="local"
|
||||
|
||||
cat <<EOF
|
||||
{
|
||||
"name": "${PKG_NAME}",
|
||||
"hierarchy": "${hierarchy}",
|
||||
"section": "${PKG_SECTION}",
|
||||
"bootstrap": "${PKG_DEPENDS_BOOTSTRAP}",
|
||||
"init": "${PKG_DEPENDS_INIT}",
|
||||
"host": "${PKG_DEPENDS_HOST}",
|
||||
"target": "${PKG_DEPENDS_TARGET}"
|
||||
},
|
||||
EOF
|
||||
done
|
||||
}
|
||||
export -f json_worker
|
||||
|
||||
start_multithread_build() {
|
||||
local buildopts bootstrap result=0
|
||||
local buildopts result=0
|
||||
|
||||
# init thread control folder
|
||||
rm -rf "${THREAD_CONTROL}"
|
||||
@ -57,11 +19,9 @@ start_multithread_build() {
|
||||
# create a single log file by default for a single threaded build (or the builder is a masochist)
|
||||
if [ ${THREADCOUNT} -eq 1 -a "${ONELOG,,}" != "no" ] || [ "${ONELOG,,}" = "yes" ]; then
|
||||
buildopts+=" --no-log-burst"
|
||||
bootstrap="&1"
|
||||
else
|
||||
mkdir -p "${THREAD_CONTROL}/logs"
|
||||
buildopts+=" --log-burst"
|
||||
bootstrap="${THREAD_CONTROL}/logs/0.log"
|
||||
fi
|
||||
buildopts+=" --log-combine ${LOGCOMBINE:-always}"
|
||||
|
||||
@ -78,20 +38,10 @@ start_multithread_build() {
|
||||
|
||||
buildopts+=" --stats-interval ${MTINTERVAL:-60}"
|
||||
|
||||
# Bootstrap GNU parallel
|
||||
if MTWITHLOCKS=no $SCRIPTS/build parallel:host 2>&1 &>${bootstrap}; then
|
||||
[ "${LOGCOMBINE}" = "always" -a -f "${bootstrap}" ] && cat "${bootstrap}"
|
||||
else
|
||||
[ "${LOGCOMBINE}" != "never" -a -f "${bootstrap}" ] && cat "${bootstrap}"
|
||||
die "Unable to bootstrap parallel package"
|
||||
fi
|
||||
|
||||
# pipefail: return value of a pipeline is the value of the last (rightmost) command to exit with a non-zero status
|
||||
set -o pipefail
|
||||
|
||||
cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \
|
||||
${TOOLCHAIN}/bin/parallel --plain --no-notice --max-args 30 --halt now,fail=1 json_worker | \
|
||||
${SCRIPTS}/genbuildplan.py --show-wants --with-json "${THREAD_CONTROL}"/plan.json \
|
||||
${SCRIPTS}/pkgjson | ${SCRIPTS}/genbuildplan.py --show-wants --with-json "${THREAD_CONTROL}"/plan.json \
|
||||
--build ${@} > "${THREAD_CONTROL}"/plan || result=1
|
||||
|
||||
if [ ${result} -eq 0 ]; then
|
||||
|
||||
55
scripts/pkgjson
Executable file
55
scripts/pkgjson
Executable file
@ -0,0 +1,55 @@
|
||||
#!/bin/bash
|
||||
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv)
|
||||
|
||||
. config/options ""
|
||||
|
||||
# This function is passed a list of package.mk paths to be processed.
|
||||
# Each package.mk is sourced with relevant variables output in JSON format.
|
||||
json_worker() {
|
||||
local packages="$@"
|
||||
local pkgpath hierarchy exited
|
||||
|
||||
exit() { exited=1; }
|
||||
|
||||
for pkgpath in ${packages}; do
|
||||
pkgpath="${pkgpath%%@*}"
|
||||
|
||||
exited=0
|
||||
if ! source_package "${pkgpath}/package.mk" &>/dev/null; then
|
||||
unset -f exit
|
||||
die "$(print_color CLR_ERROR "FAILURE: sourcing package ${pkgpath}/package.mk")"
|
||||
fi
|
||||
|
||||
[ ${exited} -eq 1 ] && continue
|
||||
|
||||
[[ ${pkgpath} =~ ^${ROOT}/${PACKAGES}/ ]] && hierarchy="global" || hierarchy="local"
|
||||
|
||||
cat <<EOF
|
||||
{
|
||||
"name": "${PKG_NAME}",
|
||||
"hierarchy": "${hierarchy}",
|
||||
"section": "${PKG_SECTION}",
|
||||
"bootstrap": "${PKG_DEPENDS_BOOTSTRAP}",
|
||||
"init": "${PKG_DEPENDS_INIT}",
|
||||
"host": "${PKG_DEPENDS_HOST}",
|
||||
"target": "${PKG_DEPENDS_TARGET}"
|
||||
},
|
||||
EOF
|
||||
done
|
||||
unset -f exit
|
||||
}
|
||||
|
||||
if [ "$1" = "--worker" ]; then
|
||||
shift
|
||||
json_worker "$*"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# pipefail: return value of a pipeline is the value of the last (rightmost) command to exit with a non-zero status
|
||||
set -o pipefail
|
||||
|
||||
cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \
|
||||
xargs --max-args=64 --max-procs="$(nproc)" "$0" --worker
|
||||
exit $?
|
||||
@ -6,24 +6,6 @@
|
||||
unset _CACHE_PACKAGE_LOCAL _CACHE_PACKAGE_GLOBAL _DEBUG_DEPENDS_LIST _DEBUG_PACKAGE_LIST
|
||||
|
||||
. config/options ""
|
||||
. config/multithread
|
||||
|
||||
# Fake the parallel command if GNU parallel is not available - slow, but works.
|
||||
fake_parallel() {
|
||||
while read -r line; do
|
||||
json_worker "${line}"
|
||||
done
|
||||
}
|
||||
|
||||
PARALLEL_BIN=${TOOLCHAIN}/bin/parallel
|
||||
|
||||
[ -x ${PARALLEL_BIN} ] || PARALLEL_BIN=parallel
|
||||
command -v ${PARALLEL_BIN} >/dev/null || PARALLEL_BIN=fake_parallel
|
||||
|
||||
# pipefail: return value of a pipeline is the value of the last (rightmost) command to exit with a non-zero status
|
||||
set -o pipefail
|
||||
|
||||
cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \
|
||||
${PARALLEL_BIN} --plain --no-notice --max-args 30 --halt now,fail=1 json_worker | \
|
||||
${SCRIPTS}/genbuildplan.py --show-wants --build ${@:-image} --warn-invalid ${GENFLAGS} || \
|
||||
${SCRIPTS}/pkgjson | ${SCRIPTS}/genbuildplan.py --show-wants --build ${@:-image} --warn-invalid ${GENFLAGS} || \
|
||||
die "FAILURE: Unable to generate plan"
|
||||
|
||||
Reference in New Issue
Block a user