0
0
mirror of https://github.com/termux/termux-packages.git synced 2025-05-10 20:55:34 +00:00
Files
termux-packages/scripts/bin/update-packages

516 lines
19 KiB
Bash
Executable File

#!/usr/bin/env bash
set -u
# Following variables should be set in environment outside of this script.
# Build updated packages.
: "${BUILD_PACKAGES:=false}"
# Github Action create issue for failed updates.
: "${CREATE_ISSUE:=false}"
# Commit changes to Git.
: "${GIT_COMMIT_PACKAGES:=false}"
# Push changes to remote.
: "${GIT_PUSH_PACKAGES:=false}"
: "${TERMUX_ARCH:="aarch64"}"
export TERMUX_PKG_UPDATE_METHOD="" # Which method to use for updating? (repology, github or gitlab)
export TERMUX_PKG_UPDATE_TAG_TYPE="" # Whether to use latest-release-tag or newest-tag.
export TERMUX_GITLAB_API_HOST="gitlab.com" # Default host for gitlab-ci.
export TERMUX_PKG_AUTO_UPDATE=false # Whether to auto-update or not. Disabled by default.
export TERMUX_PKG_UPDATE_VERSION_REGEXP="" # Regexp to extract version with `grep -oP`.
export TERMUX_PKG_UPDATE_VERSION_SED_REGEXP="" # Regexp to extract version with `sed`.
export TERMUX_REPOLOGY_DATA_FILE
TERMUX_REPOLOGY_DATA_FILE="$(mktemp -t termux-repology.XXXXXX)" # File to store repology data.
export TERMUX_SCRIPTDIR
TERMUX_SCRIPTDIR=$(realpath "$(dirname "$0")/../..") # Root of repository.
export TERMUX_PACKAGES_DIRECTORIES
TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}"/repo.json)
# Define few more variables used by scripts.
# shellcheck source=scripts/properties.sh
. "${TERMUX_SCRIPTDIR}/scripts/properties.sh"
# Utility function to write error message to stderr.
# shellcheck source=scripts/updates/utils/termux_error_exit.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_error_exit.sh
# Utility function to write updated version to build.sh.
# shellcheck source=scripts/updates/utils/termux_pkg_upgrade_version.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_pkg_upgrade_version.sh
# Utility function to check if package needs to be updated, based on version comparison.
# shellcheck source=scripts/updates/utils/termux_pkg_is_update_needed.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_pkg_is_update_needed.sh
# Wrapper around github api to get latest release or newest tag.
# shellcheck source=scripts/updates/api/termux_github_api_get_tag.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/api/termux_github_api_get_tag.sh
# Wrapper around gitlab api to get latest release or newest tag.
# shellcheck source=scripts/updates/api/termux_gitlab_api_get_tag.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/api/termux_gitlab_api_get_tag.sh
# Function to get latest version of a package as per repology.
# shellcheck source=scripts/updates/api/termux_repology_api_get_latest_version.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/api/termux_repology_api_get_latest_version.sh
# Default auto update script for packages hosted on github.com. Should not be overrided by build.sh.
# To use custom algorithm, one should override termux_pkg_auto_update().
# shellcheck source=scripts/updates/internal/termux_github_auto_update.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_github_auto_update.sh
# Default auto update script for packages hosted on hosts using gitlab-ci. Should not be overrided by build.sh.
# To use custom algorithm, one should override termux_pkg_auto_update().
# shellcheck source=scripts/updates/internal/termux_gitlab_auto_update.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_gitlab_auto_update.sh
# Default auto update script for rest packages. Should not be overrided by build.sh.
# To use custom algorithm, one should override termux_pkg_auto_update().
# shellcheck source=scripts/updates/internal/termux_repology_auto_update.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_repology_auto_update.sh
# shellcheck source=scripts/updates/internal/termux_github_graphql.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_github_graphql.sh
# Main script to:
# - by default, decide which update method to use,
# - but can be overridden by build.sh to use custom update method.
# - For example: see cmake's build.sh.
# shellcheck source=scripts/updates/termux_pkg_auto_update.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/termux_pkg_auto_update.sh
# Converts milliseconds to human-readable format.
# Example: `ms_to_human_readable 123456789` => 34h 17m 36s 789ms
ms_to_human_readable() {
echo "$(($1/3600000))h $(($1%3600000/60000))m $(($1%60000/1000))s $(($1%1000))ms" | sed 's/0h //;s/0m //;s/0s //'
}
# Runs a command without displaying its output in the case if trace is disabled and displays output in the case if it is enabled.
# Needed only for debugging
quiet() {
if [[ "$-" =~ x ]]; then
"$@"
else
&>/dev/null "$@"
fi
return $?
}
_update() {
export TERMUX_PKG_NAME
TERMUX_PKG_NAME="$(basename "$1")"
export TERMUX_PKG_BUILDER_DIR
TERMUX_PKG_BUILDER_DIR="$(realpath "$1")" # Directory containing build.sh.
IFS="," read -r -a EXCLUDED_ARCH <<<"${TERMUX_PKG_EXCLUDED_ARCHES:-}"
export TERMUX_ARCH="" # Arch to test updates.
for arch in aarch64 arm i686 x86_64; do
# shellcheck disable=SC2076
if [[ ! " ${EXCLUDED_ARCH[*]} " =~ " ${arch} " ]]; then
TERMUX_ARCH="${arch}"
break
fi
done
# Set +e +u to avoid:
# - ending on errors such as $(which prog), where prog is not installed.
# - error on unbound variable. (All global variables used should be covered by properties.sh and above.)
set +e +u
# shellcheck source=/dev/null
. "${TERMUX_PKG_BUILDER_DIR}"/build.sh 2>/dev/null
set -e -u
echo # Newline.
echo "INFO: Updating ${TERMUX_PKG_NAME} [Current version: ${TERMUX_PKG_VERSION}]"
termux_pkg_auto_update
}
declare -A _LATEST_TAGS=()
declare -A _FAILED_UPDATES=()
declare -a _ALREADY_SEEN=() # Array of packages successfully updated or skipped.
# _fetch_and_cache_tags fetches all possible tags using termux_pkg_auto_update, but using Ninja build system.
# The key difference is that we make the process concurrent, allowing us to fetch tags simultaneously rather than one at a time.
# Once all tags are cached, the termux_pkg_auto_update function will operate much more quickly.
# We avoid packages with overwritten termux_pkg_auto_update to prevent unexpected modifications to the package`s build.sh.
# shellcheck disable=SC2015
_fetch_and_cache_tags() {
echo "INFO: Fetching and caching tags"
# First invocation of termux_repology_api_get_latest_version fetches and caches repology metadata.
quiet termux_repology_api_get_latest_version ' '
local __PACKAGES=()
local __GITHUB_PACKAGES=()
# Find all packages to be processed
while read -r pkg_dir; do
BUILD_PACKAGES=false quiet _should_update "${pkg_dir}" || continue # Skip if not needed.
__PACKAGES+=("${pkg_dir}")
# bash-builtin version of `grep '^TERMUX_PKG_SRCURL=' "${pkg_dir}/build.sh" | grep -q 'github.com'` without spawning extra processes
[[ "$(<"$pkg_dir/build.sh")" =~ TERMUX_PKG_SRCURL=[^[:space:]]*github\.com ]] && __GITHUB_PACKAGES+=("$pkg_dir") || :
# Find all build.sh files and explicitly filter only files without custom `termux_pkg_auto_update`
done < <(grep -rL '^termux_pkg_auto_update' $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json") --include=build.sh | sed 's#/build\.sh$##')
echo "INFO: Building GraphQL queries"
local GITHUB_GRAPHQL_QUERIES=() COUNTER=0
declare -A __GITHUB_GRAPHQL_PACKAGES=()
# Ignore non-constant sources
# shellcheck disable=SC1091
for pkg_dir in "${__GITHUB_PACKAGES[@]}"; do
local PKG_SRCURL TAG_TYPE OWNER REPO
read -r PKG_SRCURL TAG_TYPE < <(
set +u
source "${pkg_dir}/build.sh"
echo "${TERMUX_PKG_SRCURL} ${TERMUX_PKG_UPDATE_TAG_TYPE}"
)
IFS=/ read -r OWNER REPO _ <<< "${PKG_SRCURL#*://github.com/}"
if [[ -z "${TAG_TYPE}" ]]; then # If not set, then decide on the basis of url.
if [[ "${PKG_SRCURL:0:4}" == "git+" ]]; then
# Get newest tag.
TAG_TYPE="newest-tag"
else
# Get the latest release tag.
TAG_TYPE="latest-release-tag"
fi
fi
# We prepare the query snippets for `termux_github_graphql` here
# since we already have the needed information available.
GITHUB_GRAPHQL_QUERIES+=( "_$((COUNTER++)): repository(owner: \\\"${OWNER}\\\", name: \\\"${REPO}\\\") { ..._${TAG_TYPE//-/_} }" )
unset PKG_SRCURL TAG_TYPE OWNER REPO
done
# This is called indirectly in subshell
# So silence shellcheck's unreachable code warning
# shellcheck disable=SC2317
__main__() {
cd "${TERMUX_SCRIPTDIR}"
export TERMUX_PKG_NAME="${1##*/}" TERMUX_PKG_BUILDER_DIR=${1}
set +eu
for i in scripts/updates/{**/,}*.sh "${1}/build.sh"; do
# shellcheck disable=SC1090
source "${i}"
done
set -eu
# PKG means regular version to be cached normally.
# SKIP means we know package is up to date and we can safely skip checks.
# LOCAL is for obtaining current package version, to be compared with version obtained with GraphQL
termux_github_api_get_tag() {
local ver="${TERMUX_PKG_VERSION#*:}"
echo "LOCAL|${TERMUX_PKG_NAME}|${ver#v}"
exit 0
}
termux_pkg_upgrade_version() {
local action; action="$( [[ "${1:-}" == "${TERMUX_PKG_VERSION#*:}" ]] && echo SKIP || echo PKG )"
[[ "${1:-}" == "LOCAL|"* ]] && { echo "$1"; exit 0; } # Forwarded from termux_github_api_get_tag
[[ -n "$1" ]] && echo "${action}|${TERMUX_PKG_NAME}|${1#*:}"
exit 0
}
termux_repology_auto_update() {
local ver; ver="$(termux_repology_api_get_latest_version "${TERMUX_PKG_NAME}")"
if [[ "${ver}" == "null" ]]; then
# it returns `null` in the case if the version is up to date so we can safely skip the package
echo "SKIP|${TERMUX_PKG_NAME}|${TERMUX_PKG_VERSION#*:}"
else
# The package needs to be updated
echo "PKG|${TERMUX_PKG_NAME}|${ver}"
fi
exit 0
}
termux_pkg_auto_update
}
echo "$([[ "${CI-false}" == "true" ]] && echo "::group::" || :)INFO: Fetching GitHub packages via GraphQL API"
LATEST_TAGS_GITHUB="$(
termux_github_graphql "${GITHUB_GRAPHQL_QUERIES[@]}"
)"
[[ "${CI-false}" == "true" ]] && echo "::endgroup::" || :
echo "INFO: Fetching non-GitHub packages"
local LATEST_TAGS=''
LATEST_TAGS="$(
export -f __main__
export TERMUX_SCRIPTDIR GITHUB_TOKEN TERMUX_REPOLOGY_DATA_FILE
printf '%s\0' "${__PACKAGES[@]}" | xargs -0 -n1 -P"$(nproc)" bash -c '__main__ "$@"' _ |& grep -E '^(LOCAL|PKG|SKIP)\|'
)"
unset -f __main__
declare -A __GITHUB_CURRENT_TAGS=()
[[ "${CI-false}" == "true" ]] && echo "::group::INFO: Skipping the following up-to-date packages" || :
while IFS='|' read -r type pkg version; do
if [[ "${type}" == "LOCAL" ]]; then # Current git-originated package version
__GITHUB_CURRENT_TAGS["${pkg:-_}"]="$version"
continue
fi
_LATEST_TAGS["${pkg:-_}"]="$version"
if [[ "${type}" == "SKIP" ]] || [[ "${type}" == "GIT" && "${__GITHUB_CURRENT_TAGS["${pkg:-}"]:-}" == "${version}" ]]; then
echo "INFO: Skipping ${pkg}: already at version ${version}"
_ALREADY_SEEN+=("${pkg}")
fi
done < <(printf '%s\n' "$LATEST_TAGS" "$LATEST_TAGS_GITHUB")
[[ "${CI-false}" == "true" ]] && echo "::endgroup::" || :
unset __GITHUB_CURRENT_TAGS
}
_check_updated() {
if [[ -n "${_LATEST_TAGS[${1##*/}]:-}" ]]; then
(
set +eu
quiet source "${1}/build.sh"
set -eu
export TERMUX_PKG_UPGRADE_VERSION_DRY_RUN=1
if quiet termux_pkg_upgrade_version "${_LATEST_TAGS[${1##*/}]}"; then
echo "INFO: Skipping ${1##*/}: already at version ${TERMUX_PKG_VERSION#*:}"
return 0
fi
return 1
)
local _ANSWER=$?
if (( _ANSWER == 0 )) ; then
_ALREADY_SEEN+=("$(basename "${1}")")
fi
return $_ANSWER
fi
return 1
}
_run_update() {
local pkg_dir="$1"
# Run each package update in separate process since we include their environment variables.
local output=""
if output=$(
set -euo pipefail
# Pass cached tag we obtained earlier
[[ -n "${_LATEST_TAGS[${pkg_dir##*/}]:-}" ]] && export __CACHED_TAG="${_LATEST_TAGS[${pkg_dir##*/}]}" || :
exec > >(tee /dev/fd/2) 2>&1 # output everything to stdout as well.
_update "${pkg_dir}"
); then
_ALREADY_SEEN+=("$(basename "${pkg_dir}")")
else
_FAILED_UPDATES["$(basename "${pkg_dir}")"]="${output}"
fi
}
declare -a _CACHED_ISSUE_TITLES=()
# Check if an issue with same title already exists and is open.
_gh_check_issue_exists() {
local pkg_name="$1"
if [[ -z "${_CACHED_ISSUE_TITLES[*]}" ]]; then
while read -r title; do
_CACHED_ISSUE_TITLES+=("'${title}'") # An extra quote ('') is added to avoid false positive matches.
done <<<"$(
gh issue list \
--limit 10000 \
--label "auto update failing" --label "bot" \
--state open \
--search "Auto update failing for in:title type:issue" \
--json title | jq -r '.[] | .title' | sort | uniq
)"
fi
# shellcheck disable=SC2076 # We want literal match here, not regex based.
if [[ "${_CACHED_ISSUE_TITLES[*]}" =~ "'Auto update failing for ${pkg_name}'" ]]; then
return 0
fi
return 1
}
_should_update() {
local pkg_dir="$1"
if [[ ! -f "${pkg_dir}/build.sh" ]]; then
# Fail if detected a non-package directory.
termux_error_exit "ERROR: directory '${pkg_dir}' is not a package."
fi
# `[[ "$(<"$file")" =~ regex ]]` is bash builtin solution for `grep -q` without spawning extra process
# it is needed to pre-fetch and filter packages as fast as possible (it is 6.5 times faster than `grep`)
case "${PACKAGES_OPT_OUT:-}" in
"true") [[ $'\n'"$(<"$pkg_dir/build.sh")"$'\n' != *$'\n'TERMUX_PKG_AUTO_UPDATE=false$'\n'* ]] || return 1 ;; # Skip.
*) [[ $'\n'"$(<"$pkg_dir/build.sh")"$'\n' == *$'\n'TERMUX_PKG_AUTO_UPDATE=true$'\n'* ]] || return 1 ;; # Skip.
esac
# shellcheck disable=SC2076
if [[ " ${_ALREADY_SEEN[*]} ${!_FAILED_UPDATES[*]} " =~ " $(basename "${pkg_dir}") " ]]; then
return 1 # Skip.
fi
if [[ "${BUILD_PACKAGES}${GITHUB_ACTIONS:-}" == "truetrue" ]] && _gh_check_issue_exists "$(basename "${pkg_dir}")"; then
echo "INFO: Skipping '$(basename "${pkg_dir}")', an update issue for it hasn't been resolved yet."
return 1
fi
return 0
}
shopt -s extglob
_update_dependencies() {
local pkg_dir="$1"
if ! grep -qE "^(TERMUX_PKG_DEPENDS|TERMUX_PKG_BUILD_DEPENDS|TERMUX_SUBPKG_DEPENDS)=" \
"${pkg_dir}"/+(build|*.subpackage).sh; then
return 0
fi
# shellcheck disable=SC2086 # Allow splitting of TERMUX_PACKAGES_DIRECTORIES.
while read -r dep dep_dir; do
if [[ -z $dep ]]; then
continue
elif [[ "${dep}" == "ERROR" ]]; then
termux_error_exit "ERROR: Obtaining update order failed for $(basename "${pkg_dir}")"
fi
_should_update "${dep_dir}" && ! _check_updated "${dep_dir}" && _run_update "${dep_dir}"
done < <("${TERMUX_SCRIPTDIR}"/scripts/buildorder.py "${pkg_dir}" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR")
}
echo "INFO: Running update for: $*"
if [[ "$1" == "@all" ]]; then
_fetch_and_cache_tags
for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
for pkg_dir in "${repo_dir}"/*; do
_unix_millis="$(date +%10s%3N)"
! _should_update "${pkg_dir}" && continue # Skip if not needed.
_check_updated "${pkg_dir}" && continue # Skip if already up-to-date.
_update_dependencies "${pkg_dir}" # Update all its dependencies first.
# NOTE: We are not checking whether dependencies were updated successfully or not.
# There is no way to know whether this package will build with current
# available verions of its dependencies or needs new ones.
# So, whatever the case may be. We just need packages to be updated in order
# and not care about anything else in between. If something fails to update,
# it will be reported by failure handling code, so no worries.
_run_update "${pkg_dir}"
echo "termux - took $(ms_to_human_readable $(( $(date +%10s%3N) - _unix_millis )))"
done
done
else
for pkg in "$@"; do
_unix_millis="$(date +%10s%3N)"
if [ ! -d "${pkg}" ]; then # If only package name is given, try to find it's directory.
for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
if [ -d "${repo_dir}/${pkg}" ]; then
pkg="${repo_dir}/${pkg}"
break
fi
done
fi
# Here `pkg` is a directory.
! _should_update "${pkg}" && continue
_update_dependencies "${pkg}"
_run_update "${pkg}"
echo "termux - took $(ms_to_human_readable $(( $(date +%10s%3N) - _unix_millis )))"
done
fi
unset _unix_millis
################################################FAILURE HANDLING#################################################
_gh_create_new_issue() {
local pkg_name="$1"
local max_body_length=65536 # Max length of the body for one request.
local issue_number
local body
local link
local assignee="${GITHUB_ACTOR:-}"
if [[ "${assignee:-termuxbot2}" == "termuxbot2" ]]; then
assignee="MrAdityaAlok" # Assign myself if termuxbot2 is the actor.
fi
if [[ "${CREATE_ISSUE}" != "true" ]]; then
echo "INFO: CREATE_ISSUE set to '${CREATE_ISSUE}'. Not creating new issue."
return 1
fi
if _gh_check_issue_exists "${pkg_name}"; then
echo "INFO: An existing update issue for '${pkg_name}' hasn't been resolved yet."
return
fi
# Extract origin URL, commit hash and builder directory and put everything together
link="$(git config --get remote.origin.url | sed -E 's|\.git$||; s|git@([^:]+):(.+)|https://\1/\2|')/blob/$(git rev-parse HEAD)/$(echo */"$1")"
body="$(
cat <<-EOF
Hi, I'm Termux 🤖.
I'm here to help you update your Termux packages.
I've tried to update the [${pkg_name}](${link}) package, but it failed.
Here's the output of the update script:
<details>
<summary>Show log</summary>
<pre lang="console">${_FAILED_UPDATES["${pkg_name}"]}</pre>
</details>
<hr>
<i>
Above error occured when I last tried to update at $(date -u +"%Y-%m-%d %H:%M:%S UTC").<br>
Run ID: <a href="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}">${GITHUB_RUN_ID}</a><br><br>
<b>Note:</b> Automatic updates will be disabled until this issue is resolved.<br>
</i>
EOF
)"
issue_number=$(
gh issue create \
--title "Auto update failing for ${pkg_name}" \
--body "${body:0:${max_body_length}}" \
--label "auto update failing" --label "bot" \
--assignee "${assignee}" |
grep -oE "[0-9]+" # Last component of the URL returned is the issue number.
)
if [ -z "${issue_number}" ]; then
echo "ERROR: Failed to create issue."
return 1
fi
echo "INFO: Created issue ${issue_number} for ${pkg_name}."
if [[ -n "${body:${max_body_length}}" ]]; then
# The body was too long, so we need to append the rest.
while true; do
body="${body:${max_body_length}}"
if [[ -z "${body}" ]]; then
break
fi
sleep 5 # Otherwise we might get rate limited.
gh issue edit "$issue_number" \
--body-file - <<<"$(
gh issue view "$issue_number" \
--json body \
--jq '.body'
)${body:0:${max_body_length}}" >/dev/null
# NOTE: we use --body-file instead of --body to avoid shell error 'argument list too long'.
done
fi
}
_handle_failure() {
echo # Newline.
if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
echo "INFO: Creating issue for failed updates...(if any)"
for pkg_name in "${!_FAILED_UPDATES[@]}"; do
_gh_create_new_issue "${pkg_name}"
done
else
echo "==> Failed updates:"
local count=0
for pkg_name in "${!_FAILED_UPDATES[@]}"; do
count=$((count + 1))
echo "${count}. ${pkg_name}"
done
exit 1
fi
}
if [[ ${#_FAILED_UPDATES[@]} -gt 0 ]]; then
_handle_failure
fi