mirror of
https://github.com/termux/termux-packages.git
synced 2024-11-23 14:56:16 +00:00
e8bf6f017f
on branches (except master) and forks * scripts(update-packages): error when fail dry run
449 lines
16 KiB
Bash
Executable File
449 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -u
|
|
|
|
# Following variables should be set in environment outside of this script.
|
|
# Build updated packages.
|
|
: "${BUILD_PACKAGES:=false}"
|
|
# Github Action create issue for failed updates.
|
|
: "${CREATE_ISSUE:=false}"
|
|
# Commit changes to Git.
|
|
: "${GIT_COMMIT_PACKAGES:=false}"
|
|
# Push changes to remote.
|
|
: "${GIT_PUSH_PACKAGES:=false}"
|
|
|
|
export TERMUX_PKG_UPDATE_METHOD="" # Which method to use for updating? (repology, github or gitlab)
|
|
export TERMUX_PKG_UPDATE_TAG_TYPE="" # Whether to use latest-release-tag or newest-tag.
|
|
export TERMUX_GITLAB_API_HOST="gitlab.com" # Default host for gitlab-ci.
|
|
export TERMUX_PKG_AUTO_UPDATE=false # Whether to auto-update or not. Disabled by default.
|
|
export TERMUX_PKG_UPDATE_VERSION_REGEXP="" # Regexp to extract version with `grep -oP`.
|
|
export TERMUX_PKG_UPDATE_VERSION_SED_REGEXP="" # Regexp to extract version with `sed`.
|
|
export TERMUX_REPOLOGY_DATA_FILE
|
|
TERMUX_REPOLOGY_DATA_FILE="$(mktemp -t termux-repology.XXXXXX)" # File to store repology data.
|
|
|
|
export TERMUX_SCRIPTDIR
|
|
TERMUX_SCRIPTDIR=$(realpath "$(dirname "$0")/../..") # Root of repository.
|
|
|
|
export TERMUX_PACKAGES_DIRECTORIES
|
|
TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}"/repo.json)
|
|
|
|
# Define few more variables used by scripts.
|
|
# shellcheck source=scripts/properties.sh
|
|
. "${TERMUX_SCRIPTDIR}/scripts/properties.sh"
|
|
|
|
# Utility function to write error message to stderr.
|
|
# shellcheck source=scripts/updates/utils/termux_error_exit.sh
|
|
. "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_error_exit.sh
|
|
|
|
# Utility function to write updated version to build.sh.
|
|
# shellcheck source=scripts/updates/utils/termux_pkg_upgrade_version.sh
|
|
. "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_pkg_upgrade_version.sh
|
|
|
|
# Utility function to check if package needs to be updated, based on version comparison.
|
|
# shellcheck source=scripts/updates/utils/termux_pkg_is_update_needed.sh
|
|
. "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_pkg_is_update_needed.sh
|
|
|
|
# Wrapper around github api to get latest release or newest tag.
|
|
# shellcheck source=scripts/updates/api/termux_github_api_get_tag.sh
|
|
. "${TERMUX_SCRIPTDIR}"/scripts/updates/api/termux_github_api_get_tag.sh
|
|
|
|
# Wrapper around gitlab api to get latest release or newest tag.
|
|
# shellcheck source=scripts/updates/api/termux_gitlab_api_get_tag.sh
|
|
. "${TERMUX_SCRIPTDIR}"/scripts/updates/api/termux_gitlab_api_get_tag.sh
|
|
|
|
# Function to get latest version of a package as per repology.
|
|
# shellcheck source=scripts/updates/api/termux_repology_api_get_latest_version.sh
|
|
. "${TERMUX_SCRIPTDIR}"/scripts/updates/api/termux_repology_api_get_latest_version.sh
|
|
|
|
# Default auto update script for packages hosted on github.com. Should not be overrided by build.sh.
|
|
# To use custom algorithm, one should override termux_pkg_auto_update().
|
|
# shellcheck source=scripts/updates/internal/termux_github_auto_update.sh
|
|
. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_github_auto_update.sh
|
|
|
|
# Default auto update script for packages hosted on hosts using gitlab-ci. Should not be overrided by build.sh.
|
|
# To use custom algorithm, one should override termux_pkg_auto_update().
|
|
# shellcheck source=scripts/updates/internal/termux_gitlab_auto_update.sh
|
|
. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_gitlab_auto_update.sh
|
|
|
|
# Default auto update script for rest packages. Should not be overrided by build.sh.
|
|
# To use custom algorithm, one should override termux_pkg_auto_update().
|
|
# shellcheck source=scripts/updates/internal/termux_repology_auto_update.sh
|
|
. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_repology_auto_update.sh
|
|
|
|
# Main script to:
|
|
# - by default, decide which update method to use,
|
|
# - but can be overrided by build.sh to use custom update method.
|
|
# - For example: see neovim-nightly's build.sh.
|
|
# shellcheck source=scripts/updates/termux_pkg_auto_update.sh
|
|
. "${TERMUX_SCRIPTDIR}"/scripts/updates/termux_pkg_auto_update.sh
|
|
|
|
# Converts milliseconds to human-readable format.
|
|
# Example: `ms_to_human_readable 123456789` => 34h 17m 36s 789ms
|
|
ms_to_human_readable() {
|
|
echo "$(($1/3600000))h $(($1%3600000/60000))m $(($1%60000/1000))s $(($1%1000))ms" | sed 's/0h //;s/0m //;s/0s //'
|
|
}
|
|
|
|
# Runs a command without displaying its output in the case if trace is disabled and displays output in the case if it is enabled.
|
|
# Needed only for debugging
|
|
quiet() {
|
|
if [[ "$-" =~ x ]]; then
|
|
"$@"
|
|
else
|
|
&>/dev/null "$@"
|
|
fi
|
|
return $?
|
|
}
|
|
|
|
_update() {
|
|
export TERMUX_PKG_NAME
|
|
TERMUX_PKG_NAME="$(basename "$1")"
|
|
export TERMUX_PKG_BUILDER_DIR
|
|
TERMUX_PKG_BUILDER_DIR="$(realpath "$1")" # Directory containing build.sh.
|
|
|
|
IFS="," read -r -a BLACKLISTED_ARCH <<<"${TERMUX_PKG_BLACKLISTED_ARCHES:-}"
|
|
export TERMUX_ARCH="" # Arch to test updates.
|
|
for arch in aarch64 arm i686 x86_64; do
|
|
# shellcheck disable=SC2076
|
|
if [[ ! " ${BLACKLISTED_ARCH[*]} " =~ " ${arch} " ]]; then
|
|
TERMUX_ARCH="${arch}"
|
|
break
|
|
fi
|
|
done
|
|
# Set +e +u to avoid:
|
|
# - ending on errors such as $(which prog), where prog is not installed.
|
|
# - error on unbound variable. (All global variables used should be covered by properties.sh and above.)
|
|
set +e +u
|
|
# shellcheck source=/dev/null
|
|
. "${TERMUX_PKG_BUILDER_DIR}"/build.sh 2>/dev/null
|
|
set -e -u
|
|
|
|
echo # Newline.
|
|
echo "INFO: Updating ${TERMUX_PKG_NAME} [Current version: ${TERMUX_PKG_VERSION}]"
|
|
termux_pkg_auto_update
|
|
}
|
|
|
|
declare -A _LATEST_TAGS=()
|
|
declare -A _FAILED_UPDATES=()
|
|
declare -a _ALREADY_SEEN=() # Array of packages successfully updated or skipped.
|
|
|
|
# _fetch_and_cache_tags fetches all possible tags using termux_pkg_auto_update, but using Ninja build system.
|
|
# The key difference is that we make the process concurrent, allowing us to fetch tags simultaneously rather than one at a time.
|
|
# Once all tags are cached, the termux_pkg_auto_update function will operate much more quickly.
|
|
# We avoid packages with overwritten termux_pkg_auto_update to prevent unexpected modifications to the package`s build.sh.
|
|
_fetch_and_cache_tags() {
|
|
if [ "$(uname -o)" = "Android" ] || [ -e "/system/bin/app_process" ]; then
|
|
if ! command -v ninja &> /dev/null; then
|
|
echo "INFO: Skipping fetching and caching tags. Package 'ninja' is not installed."
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if ! command -v ninja &> /dev/null; then
|
|
echo "INFO: Fetching ninja build system"
|
|
. "${TERMUX_SCRIPTDIR}"/scripts/build/termux_download.sh
|
|
. "${TERMUX_SCRIPTDIR}"/scripts/build/setup/termux_setup_ninja.sh
|
|
TERMUX_ON_DEVICE_BUILD=false TERMUX_COMMON_CACHEDIR=/tmp TERMUX_PKG_TMPDIR=/tmp termux_setup_ninja
|
|
fi
|
|
|
|
echo "INFO: Fetching and caching tags"
|
|
|
|
# First invocation of termux_repology_api_get_latest_version fetches and caches repology metadata.
|
|
quiet termux_repology_api_get_latest_version ' '
|
|
|
|
local __PACKAGES=()
|
|
for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
|
|
for pkg_dir in "${repo_dir}"/*; do
|
|
! quiet _should_update "${pkg_dir}" && continue # Skip if not needed.
|
|
grep -q '^termux_pkg_auto_update' "${pkg_dir}/build.sh" && continue # Skip if package has custom auto-update
|
|
__PACKAGES+=("${pkg_dir}")
|
|
done
|
|
done
|
|
|
|
__main__() {
|
|
cd ${TERMUX_SCRIPTDIR}
|
|
export TERMUX_PKG_NAME="${1##*/}" TERMUX_PKG_BUILDER_DIR=${1}
|
|
set +eu;
|
|
for i in scripts/updates/{**/,}*.sh "${1}/build.sh"; do source ${i}; done
|
|
set -eu
|
|
termux_pkg_upgrade_version() {
|
|
[[ -n "$1" ]] && echo "PKG|$TERMUX_PKG_NAME|${1#*:}"
|
|
exit 0
|
|
}
|
|
termux_repology_auto_update() {
|
|
echo "PKG|$TERMUX_PKG_NAME|$(termux_repology_api_get_latest_version "${TERMUX_PKG_NAME}" | sed "s/^null$/${TERMUX_PKG_VERSION#*:}/")"
|
|
exit 0
|
|
}
|
|
termux_pkg_auto_update
|
|
}
|
|
|
|
__generate__() {
|
|
echo "rule update"
|
|
echo " command = bash -c \"\$\$F\" -- \$pkg_dir ||:"
|
|
sed 's/[^ ]\+/\nbuild &: update\n pkg_dir=&/g' <<< "${__PACKAGES[@]}"
|
|
echo "build run_all: phony ${__PACKAGES[@]}"
|
|
echo "default run_all"
|
|
}
|
|
|
|
local LATEST_TAGS="$(\
|
|
F="$(declare -p TERMUX_SCRIPTDIR GITHUB_TOKEN TERMUX_REPOLOGY_DATA_FILE); $(declare -f __main__ | sed 's/__main__ ()//')" \
|
|
env --chdir=/tmp ninja -f /dev/stdin <<< "$(__generate__)" |& grep "^PKG|"
|
|
)"
|
|
|
|
unset -f __main__ __generate__
|
|
while IFS='|' read -r _ pkg version; do
|
|
_LATEST_TAGS["${pkg:-_}"]="$version"
|
|
done <<< "$LATEST_TAGS"
|
|
quiet declare -p _LATEST_TAGS
|
|
}
|
|
|
|
_check_updated() {
|
|
if [[ -n "${_LATEST_TAGS[${1##*/}]:-}" ]]; then
|
|
(
|
|
set +eu
|
|
quiet source "${1}/build.sh"
|
|
set -eu
|
|
export TERMUX_PKG_UPGRADE_VERSION_DRY_RUN=1
|
|
if quiet termux_pkg_upgrade_version "${_LATEST_TAGS[${1##*/}]}"; then
|
|
echo "INFO: Skipping ${1##*/}: already at version ${TERMUX_PKG_VERSION#*:}"
|
|
return 0
|
|
fi
|
|
return 1
|
|
)
|
|
local _ANSWER=$?
|
|
if (( _ANSWER == 0 )) ; then
|
|
_ALREADY_SEEN+=("$(basename "${1}")")
|
|
fi
|
|
return $_ANSWER
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
_run_update() {
|
|
local pkg_dir="$1"
|
|
[[ -n "${_LATEST_TAGS[${pkg_dir##*/}]:-}" ]] && export __CACHED_TAG="${_LATEST_TAGS[${pkg_dir##*/}]}"
|
|
# Run each package update in separate process since we include their environment variables.
|
|
local output=""
|
|
{
|
|
output=$(
|
|
set -euo pipefail
|
|
_update "${pkg_dir}" 2>&1 | tee /dev/fd/3 # fd 3 is used to output to stdout as well.
|
|
exit "${PIPESTATUS[0]}" # Return exit code of _update.
|
|
)
|
|
} 3>&1
|
|
# shellcheck disable=SC2181
|
|
if [[ $? -ne 0 ]]; then
|
|
_FAILED_UPDATES["$(basename "${pkg_dir}")"]="${output}"
|
|
else
|
|
_ALREADY_SEEN+=("$(basename "${pkg_dir}")")
|
|
fi
|
|
unset __CACHED_TAG
|
|
}
|
|
|
|
declare -a _CACHED_ISSUE_TITLES=()
|
|
# Check if an issue with same title already exists and is open.
|
|
_gh_check_issue_exists() {
|
|
local pkg_name="$1"
|
|
if [[ -z "${_CACHED_ISSUE_TITLES[*]}" ]]; then
|
|
while read -r title; do
|
|
_CACHED_ISSUE_TITLES+=("'${title}'") # An extra quote ('') is added to avoid false positive matches.
|
|
done <<<"$(
|
|
gh issue list \
|
|
--limit 10000 \
|
|
--label "auto update failing" --label "bot" \
|
|
--state open \
|
|
--search "Auto update failing for in:title type:issue" \
|
|
--json title | jq -r '.[] | .title' | sort | uniq
|
|
)"
|
|
fi
|
|
# shellcheck disable=SC2076 # We want literal match here, not regex based.
|
|
if [[ "${_CACHED_ISSUE_TITLES[*]}" =~ "'Auto update failing for ${pkg_name}'" ]]; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
_should_update() {
|
|
local pkg_dir="$1"
|
|
|
|
if [[ ! -f "${pkg_dir}/build.sh" ]]; then
|
|
# Fail if detected a non-package directory.
|
|
termux_error_exit "ERROR: directory '${pkg_dir}' is not a package."
|
|
fi
|
|
|
|
if { [[ "${PACKAGES_OPT_OUT:-}" == "true" ]] && grep -q '^TERMUX_PKG_AUTO_UPDATE=false$' "${pkg_dir}/build.sh"; } || \
|
|
{ [[ "${PACKAGES_OPT_OUT:-}" != "true" ]] && ! grep -q '^TERMUX_PKG_AUTO_UPDATE=true$' "${pkg_dir}/build.sh"; }; then
|
|
return 1 # Skip.
|
|
fi
|
|
|
|
# shellcheck disable=SC2076
|
|
if [[ " ${_ALREADY_SEEN[*]} ${!_FAILED_UPDATES[*]} " =~ " $(basename "${pkg_dir}") " ]]; then
|
|
return 1 # Skip.
|
|
fi
|
|
if [[ "${BUILD_PACKAGES}" == "true" ]] && [[ "${GITHUB_ACTIONS:-}" == "true" ]] && _gh_check_issue_exists "$(basename "${pkg_dir}")"; then
|
|
echo "INFO: Skipping '$(basename "${pkg_dir}")', an update issue for it hasn't been resolved yet."
|
|
return 1
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
shopt -s extglob
|
|
_update_dependencies() {
|
|
local pkg_dir="$1"
|
|
|
|
if ! grep -qE "^(TERMUX_PKG_DEPENDS|TERMUX_PKG_BUILD_DEPENDS|TERMUX_SUBPKG_DEPENDS)=" \
|
|
"${pkg_dir}"/+(build|*.subpackage).sh; then
|
|
return 0
|
|
fi
|
|
# shellcheck disable=SC2086 # Allow splitting of TERMUX_PACKAGES_DIRECTORIES.
|
|
while read -r dep dep_dir; do
|
|
if [[ -z $dep ]]; then
|
|
continue
|
|
elif [[ "${dep}" == "ERROR" ]]; then
|
|
termux_error_exit "ERROR: Obtaining update order failed for $(basename "${pkg_dir}")"
|
|
fi
|
|
_should_update "${dep_dir}" && ! _check_updated "${dep_dir}" && _run_update "${dep_dir}"
|
|
done <<<"$("${TERMUX_SCRIPTDIR}"/scripts/buildorder.py "${pkg_dir}" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR")"
|
|
}
|
|
|
|
echo "INFO: Running update for: $*"
|
|
|
|
if [[ "$1" == "@all" ]]; then
|
|
_fetch_and_cache_tags
|
|
for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
|
|
for pkg_dir in "${repo_dir}"/*; do
|
|
_unix_millis=$(date +%s%N | cut -b1-13)
|
|
! _should_update "${pkg_dir}" && continue # Skip if not needed.
|
|
_check_updated "${pkg_dir}" && continue # Skip if already up-to-date.
|
|
# Update all its dependencies first.
|
|
_update_dependencies "${pkg_dir}"
|
|
# NOTE: I am not cheacking whether dependencies were updated successfully or not.
|
|
# There is no way I could know whether this package will build with current
|
|
# available verions of its dependencies or needs new ones.
|
|
# So, whatever the case may be. We just need packages to be updated in order
|
|
# and not care about anything else in between. If something fails to update,
|
|
# it will be reported by failure handling code, so no worries.
|
|
_run_update "${pkg_dir}"
|
|
echo "termux - took $(ms_to_human_readable $(( $(date +%s%N | cut -b1-13) - _unix_millis )))"
|
|
done
|
|
done
|
|
else
|
|
for pkg in "$@"; do
|
|
if [ ! -d "${pkg}" ]; then # If only package name is given, try to find it's directory.
|
|
for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
|
|
if [ -d "${repo_dir}/${pkg}" ]; then
|
|
pkg="${repo_dir}/${pkg}"
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
# Here `pkg` is a directory.
|
|
! _should_update "${pkg}" && continue
|
|
_update_dependencies "${pkg}"
|
|
_run_update "${pkg}"
|
|
done
|
|
fi
|
|
|
|
################################################FAILURE HANDLING#################################################
|
|
|
|
_gh_create_new_issue() {
|
|
local pkg_name="$1"
|
|
local max_body_length=65536 # Max length of the body for one request.
|
|
local issue_number
|
|
local body
|
|
local link
|
|
local assignee="${GITHUB_ACTOR:-}"
|
|
if [[ "${assignee:-termuxbot2}" == "termuxbot2" ]]; then
|
|
assignee="MrAdityaAlok" # Assign myself if termuxbot2 is the actor.
|
|
fi
|
|
|
|
if [[ "${CREATE_ISSUE}" != "true" ]]; then
|
|
echo "INFO: CREATE_ISSUE set to '${CREATE_ISSUE}'. Not creating new issue."
|
|
return 1
|
|
fi
|
|
if _gh_check_issue_exists "${pkg_name}"; then
|
|
echo "INFO: An existing update issue for '${pkg_name}' hasn't been resolved yet."
|
|
return
|
|
fi
|
|
|
|
# Extract origin URL, commit hash and builder directory and put everything together
|
|
link="$(git config --get remote.origin.url |sed -E 's|\.git$||; s|git@([^:]+):(.+)|https://\1/\2|')/blob/$(git rev-parse HEAD)/$(echo */$1)"
|
|
|
|
body="$(
|
|
cat <<-EOF
|
|
Hi, I'm Termux 🤖.
|
|
|
|
I'm here to help you update your Termux packages.
|
|
|
|
I've tried to update the [${pkg_name}](${link}) package, but it failed.
|
|
|
|
Here's the output of the update script:
|
|
<details>
|
|
<summary>Show log</summary>
|
|
<pre lang="console">${_FAILED_UPDATES["${pkg_name}"]}</pre>
|
|
</details>
|
|
|
|
<hr>
|
|
<i>
|
|
Above error occured when I last tried to update at $(date -u +"%Y-%m-%d %H:%M:%S UTC").<br>
|
|
Run ID: <a href="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}">${GITHUB_RUN_ID}</a><br><br>
|
|
<b>Note:</b> Automatic updates will be disabled until this issue is resolved.<br>
|
|
</i>
|
|
EOF
|
|
)"
|
|
issue_number=$(
|
|
gh issue create \
|
|
--title "Auto update failing for ${pkg_name}" \
|
|
--body "${body:0:${max_body_length}}" \
|
|
--label "auto update failing" --label "bot" \
|
|
--assignee ${assignee} |
|
|
grep -oE "[0-9]+" # Last component of the URL returned is the issue number.
|
|
)
|
|
if [ -z "${issue_number}" ]; then
|
|
echo "ERROR: Failed to create issue."
|
|
return 1
|
|
fi
|
|
|
|
echo "INFO: Created issue ${issue_number} for ${pkg_name}."
|
|
|
|
if [[ -n "${body:${max_body_length}}" ]]; then
|
|
# The body was too long, so we need to append the rest.
|
|
while true; do
|
|
body="${body:${max_body_length}}"
|
|
if [[ -z "${body}" ]]; then
|
|
break
|
|
fi
|
|
sleep 5 # Otherwise we might get rate limited.
|
|
gh issue edit "$issue_number" \
|
|
--body-file - <<<"$(
|
|
gh issue view "$issue_number" \
|
|
--json body \
|
|
--jq '.body'
|
|
)${body:0:${max_body_length}}" >/dev/null
|
|
# NOTE: we use --body-file instead of --body to avoid shell error 'argument list too long'.
|
|
done
|
|
fi
|
|
}
|
|
|
|
_handle_failure() {
|
|
echo # Newline.
|
|
if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
|
|
echo "INFO: Creating issue for failed updates...(if any)"
|
|
for pkg_name in "${!_FAILED_UPDATES[@]}"; do
|
|
_gh_create_new_issue "${pkg_name}"
|
|
done
|
|
else
|
|
echo "==> Failed updates:"
|
|
local count=0
|
|
for pkg_name in "${!_FAILED_UPDATES[@]}"; do
|
|
count=$((count + 1))
|
|
echo "${count}. ${pkg_name}"
|
|
done
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
if [[ ${#_FAILED_UPDATES[@]} -gt 0 ]]; then
|
|
_handle_failure
|
|
fi
|