DistroHopper/generate
2026-05-27 17:00:06 +02:00

616 lines
18 KiB
Bash
Executable file

#!/usr/bin/env bash
# shellcheck disable=SC2317
# shellcheck source=lib.sh
SCRIPT_DIR="$(dirname "$(realpath "$0")")"
source "${SCRIPT_DIR}/lib.sh"
export PUBLIC_DIR="${PUBLIC_DIR:-${SCRIPT_DIR}/public}"
TEMPLATES_DIR="${TEMPLATES_DIR:-${SCRIPT_DIR}/templates}"
generateVersion="0.6"
check_hash() {
local iso=""
local hash="$2"
local status="OK"
if [ "${OPERATION}" == "download" ]; then
iso="${1}"
else
iso="${VM_PATH}/${1}"
fi
local hash_algo
case ${#hash} in
32) hash_algo=md5sum ;;
40) hash_algo=sha1sum ;;
64) hash_algo=sha256sum ;;
128) hash_algo=sha512sum ;;
*) status="UNKNOWN" ;;
esac
if [[ "$status" != "UNKNOWN" ]]; then
if ! echo "${hash} ${iso}" | ${hash_algo} --check --status; then
status="FAIL"
fi
fi
if [[ "$status" == "FAIL" ]]; then
echo $"ERROR!"
exit 1
fi
}
function write_output() {
unset -f config_ editions_ arch_ get_ releases_ extract_
unset GUEST IMAGE_TYPE DW_ARCHITECTURE DW_CATEGORY DW_DESKTOP DW_URL ORIGIN EDITIONS
. "${TEMPLATES_DIR}/${OS}"
local DAT="${PUBLIC_DIR}/tmp_${OS}.dat"
if [ ! -s "${DAT}" ]; then
echo "WARNING: no entries collected for ${OS}, skipping"
return 1
fi
# In test mode, only PASS rows count; otherwise everything emitted is OK.
local _filter='OK'
[ "${OPERATION}" == "test" ] && _filter='PASS'
# Collect releases/editions per arch from structured tmp_${OS}.dat
declare -A _arch_releases=() _arch_editions=()
local _all_archs="" _arch _rel _ed _url _st
while IFS=$'\t' read -r _arch _rel _ed _url _st; do
[[ "${_st}" == "${_filter}" ]] || continue
[ -n "${_arch}" ] || continue
[[ " ${_all_archs} " == *" ${_arch} "* ]] || _all_archs+=" ${_arch}"
_arch_releases[${_arch}]+=" ${_rel}"
[ "${_ed}" != "-" ] && [ -n "${_ed}" ] && _arch_editions[${_arch}]+=" ${_ed}"
done < "${DAT}"
_all_archs=$(printf '%s\n' ${_all_archs} | sort -u | grep -v '^$' | paste -sd ' ')
[ -z "${_all_archs}" ] && { echo "WARNING: no PASS entries for ${OS}, skipping"; return 1; }
# Default sets = union across archs
local _default_rel _default_ed _all_rel="" _all_ed=""
local _a
for _a in ${_all_archs}; do
_all_rel+=" ${_arch_releases[${_a}]}"
_all_ed+=" ${_arch_editions[${_a}]}"
done
_default_rel=$(printf '%s\n' ${_all_rel} | sort -ur | grep -v '^$' | paste -sd ' ')
_default_ed=$(printf '%s\n' ${_all_ed} | sort -u | grep -v '^$' | paste -sd ' ')
# Per-arch overrides (only emitted when they differ from default)
local _override_lines=""
for _a in ${_all_archs}; do
local _ar _ae
_ar=$(printf '%s\n' ${_arch_releases[${_a}]} | sort -ur | grep -v '^$' | paste -sd ' ')
_ae=$(printf '%s\n' ${_arch_editions[${_a}]} | sort -u | grep -v '^$' | paste -sd ' ')
if [ "${_ar}" != "${_default_rel}" ]; then
_override_lines+="RELEASES_${_a}=\"${_ar}\""$'\n'
fi
if [ -n "${_default_ed}" ] && [ "${_ae}" != "${_default_ed}" ]; then
_override_lines+="EDITIONS_${_a}=\"${_ae}\""$'\n'
fi
done
{
printf '# Template file for '\''%s'\''\n' "$OS"
printf 'OSNAME="%s"\n' "${OSNAME:-${OS}}"
printf 'PRETTY="%s"\n' "${PRETTY:-}"
printf 'LOGO="%s"\n' "${LOGO:-}"
printf 'ICON="%s"\n' "${ICON:-}"
printf 'ICON_ONLINE="%s"\n' "${ICON_ONLINE:-}"
printf 'CATEGORY="%s"\n' "${CATEGORY:-}"
printf 'BASEDOF="%s"\n' "${BASEDOF:-}"
printf 'DESCRIPTION="%s"\n' "${DESCRIPTION:-}"
printf 'HOMEPAGE="%s"\n' "${HOMEPAGE:-}"
printf 'CREDENTIALS="%s"\n' "${CREDENTIALS:-}"
printf 'GPG="%s"\n' "${GPG:-}"
printf 'RSS="%s"\n' "${RSS:-}"
printf 'DW="%s"\n' "${DW:-}"
printf 'MAGNET="%s"\n' "${MAGNET:-}"
printf 'CHAT="%s"\n' "${CHAT:-}"
for _key in GUEST IMAGE_TYPE DW_ARCHITECTURE DW_CATEGORY DW_DESKTOP DW_URL ORIGIN; do
local _val="${!_key}"
[ -n "${_val}" ] && printf '%s="%s"\n' "${_key}" "${_val}"
done
echo
printf 'RELEASES="%s"\n' "${_default_rel}"
[ -n "${_default_ed}" ] && printf 'EDITIONS="%s"\n' "${_default_ed}"
printf 'QEMU_ARCH="%s"\n' "${_all_archs}"
if [ -n "${_override_lines}" ]; then
echo
printf '%s' "${_override_lines}"
fi
# Embed all template functions verbatim so public/<OS> is self-contained.
# get_entries_ is excluded — it's a build-time helper from lib.sh.
while IFS= read -r _decl; do
local _fn="${_decl#function }"; _fn="${_fn%%(*}"
[[ "${_fn}" == "get_entries_" ]] && continue
echo
sed -n "/^${_decl}/,/^}/p" "${TEMPLATES_DIR}/${OS}" | sed '1s/^function //'
done < <(grep "^function " "${TEMPLATES_DIR}/${OS}")
} >"${PUBLIC_DIR}/${OS}"
echo "wrote ${PUBLIC_DIR}/${OS}"
}
function is_arch_supported() {
local OS="${1}"
local CHECK_ARCH="${2}"
local SUPPORTED=""
if [[ $(type -t "arch_") == function ]]; then
SUPPORTED=$(arch_)
else
SUPPORTED="amd64"
fi
[[ " ${SUPPORTED} " == *" ${CHECK_ARCH} "* ]]
}
function _iterate_os() {
OS="${1}"
mkdir -p "${PUBLIC_DIR}"
rm -f "${PUBLIC_DIR}/${OS}" "${PUBLIC_DIR}/tmp_${OS}" "${PUBLIC_DIR}/tmp_${OS}.dat"
touch "${PUBLIC_DIR}/tmp_${OS}" "${PUBLIC_DIR}/tmp_${OS}.dat"
# Per-OS HTTP cache — exported so child shells / subshells see it.
# Wiped on RETURN so each OS gets a fresh cache.
export _WEB_CACHE_DIR
_WEB_CACHE_DIR="$(mktemp -d -t "dh-webcache-${OS}-XXXXXX")"
trap 'rm -rf "${_WEB_CACHE_DIR}"; unset _WEB_CACHE_DIR' RETURN
unset -f get_ releases_ editions_ arch_ config_ extract_
. "${TEMPLATES_DIR}/${OS}"
local ARCHS
if [[ $(type -t "arch_") == function ]]; then
ARCHS=$(arch_)
else
ARCHS="amd64"
fi
# Cache releases per arch once (releases_ may depend on ARCH/EDITION)
for ARCH in ${ARCHS}; do
for RELEASE in $(releases_); do
. "${TEMPLATES_DIR}/${OS}"
get_entries_ "${OS}" "${ARCH}" "${RELEASE}" "${OPERATION}" "${PUBLIC_DIR}" "${TEMPLATES_DIR}"
done
done
}
# Verify a checksum file against a GPG signature.
#
# Usage: verify_gpg SUMS_URL KEY_SOURCE
# SUMS_URL — URL of the SHA256SUMS / MD5SUMS file already used for hash check
# KEY_SOURCE — value of GPG="" from the template:
# "" → skip (no GPG configured)
# "FINGERPRINT" → fetch key from keyserver
# "https://..." → download key from URL
#
# Signature URL is derived as SUMS_URL + ".asc" (fallback: ".gpg").
function verify_gpg() {
local SUMS_URL="${1}"
local KEY_SOURCE="${2:-${GPG:-}}"
local SIG_FILE="" KEY_FILE="" SUMS_FILE="" GPG_HOME=""
if [ -z "${KEY_SOURCE}" ]; then
return 0
fi
if ! command -v gpg &>/dev/null; then
echo "WARNING: gpg not found — skipping signature verification"
return 1
fi
SIG_FILE=$(mktemp)
SUMS_FILE=$(mktemp)
GPG_HOME=$(mktemp -d)
chmod 700 "${GPG_HOME}"
# Download SUMS file
curl --disable --silent --location -o "${SUMS_FILE}" "${SUMS_URL}" 2>/dev/null || {
echo "WARNING: Failed to download SUMS file for GPG check — ${SUMS_URL}"
rm -f "${SIG_FILE}" "${SUMS_FILE}"; rm -rf "${GPG_HOME}"
return 1
}
# Download signature (.asc preferred, fallback .gpg)
if ! curl --disable --silent --location --fail -o "${SIG_FILE}" "${SUMS_URL}.asc" 2>/dev/null; then
if ! curl --disable --silent --location --fail -o "${SIG_FILE}" "${SUMS_URL}.gpg" 2>/dev/null; then
echo "WARNING: No GPG signature found at ${SUMS_URL}.asc or .gpg"
rm -f "${SIG_FILE}" "${SUMS_FILE}"; rm -rf "${GPG_HOME}"
return 1
fi
fi
# Import signing key
KEY_FILE=$(mktemp)
if [[ "${KEY_SOURCE}" =~ ^https?:// ]]; then
curl --disable --silent --location -o "${KEY_FILE}" "${KEY_SOURCE}" 2>/dev/null || {
echo "WARNING: Failed to download GPG key — ${KEY_SOURCE}"
rm -f "${SIG_FILE}" "${SUMS_FILE}" "${KEY_FILE}"; rm -rf "${GPG_HOME}"
return 1
}
GNUPGHOME="${GPG_HOME}" gpg --quiet --import "${KEY_FILE}" 2>/dev/null
else
# Treat as fingerprint — fetch from keyserver
GNUPGHOME="${GPG_HOME}" gpg --quiet --keyserver hkps://keyserver.ubuntu.com \
--recv-keys "${KEY_SOURCE}" 2>/dev/null || \
GNUPGHOME="${GPG_HOME}" gpg --quiet --keyserver hkps://keys.openpgp.org \
--recv-keys "${KEY_SOURCE}" 2>/dev/null
fi
local result=0
if GNUPGHOME="${GPG_HOME}" gpg --verify "${SIG_FILE}" "${SUMS_FILE}" 2>/dev/null; then
echo "GPG OK — ${PRETTY:-${OS}} ${RELEASE}"
else
echo "WARNING: GPG verification FAILED — ${PRETTY:-${OS}} ${RELEASE}"
result=1
fi
rm -f "${SIG_FILE}" "${SUMS_FILE}" "${KEY_FILE}"; rm -rf "${GPG_HOME}"
return ${result}
}
# Download a file from the web and pipe it to stdout
function web_pipe() {
local URL="${1}"
local key body
key=$(_web_cache_key GET "${URL}")
if body=$(_web_cache_get_body "${key}"); then
printf '%s' "${body}"
return 0
fi
body=$(curl --disable --silent --location "${URL}")
printf '%s' "${body}" | _web_cache_put_body "${key}"
printf '%s' "${body}"
}
# Download a JSON file from the web and pipe it to stdout
function web_pipe_json() {
local URL="${1}"
local key body
key=$(_web_cache_key GET_JSON "${URL}")
if body=$(_web_cache_get_body "${key}"); then
printf '%s' "${body}"
return 0
fi
body=$(curl --disable --silent --location --header "Accept: application/json" "${URL}")
printf '%s' "${body}" | _web_cache_put_body "${key}"
printf '%s' "${body}"
}
# checks if a URL is reachable
function web_check() {
local HEADERS=()
local URL="${1}"
# Process any headers
while (("$#")); do
if [ "${1}" == "--header" ]; then
HEADERS+=("${1}" "${2}")
shift 2
else
shift
fi
done
local key rc
key=$(_web_cache_key "HEAD${HEADERS[*]}" "${URL}")
if _web_cache_get_status "${key}"; then return 0; fi
rc=$?
# rc=2 means not cached; otherwise it's the cached exit status
if [ "${rc}" != "2" ]; then return "${rc}"; fi
curl --silent --location --head --output /dev/null --fail --connect-timeout 30 --max-time 30 --retry 3 "${HEADERS[@]}" "${URL}"
rc=$?
_web_cache_put_status "${key}" "${rc}"
return "${rc}"
}
# checks if a URL needs to be redirected and returns the final URL
function web_redirect() {
local REDIRECT_URL=""
local URL="${1}"
# Use HEAD to follow redirects without downloading the body.
REDIRECT_URL=$(curl --silent --location --head --max-time 10 --write-out '%{url_effective}' --output /dev/null "${URL}" 2>/dev/null)
if [ -n "${REDIRECT_URL}" ] && [ "${REDIRECT_URL}" != "${URL}" ]; then
echo "${REDIRECT_URL}"
else
echo "${URL}"
fi
}
function _record_test_result() {
local URL="${1}"
local CHECK
CHECK=$(web_check "${URL}" && echo "PASS" || echo "FAIL")
table_add_row "$OS" "${ARCH:-amd64}" "$RELEASE" "$EDITION" "$URL" "$CHECK"
printf "%-10s %-15s %-15s %-10s %-30s %-10s\n" "${OS}" "${ARCH:-amd64}" "${RELEASE}" "${EDITION}" "${URL}" "${CHECK}" | tee -a "${PUBLIC_DIR}/tmp_${OS}"
}
# Download a file from the web
function web_get() {
local HEADERS=()
local URL="${1}"
local DIR="${2}"
local FILE=""
local USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
if [ -n "${3}" ]; then
FILE="${3}"
else
FILE="${URL##*/}"
fi
# Process any URL redirections after the file name has been extracted
URL=$(web_redirect "${URL}")
# Process any headers
while (("$#")); do
if [ "${1}" == "--header" ]; then
HEADERS+=("${1}" "${2}")
shift 2
else
shift
fi
done
if [ "${OPERATION}" == "show" ]; then
printf "%-10s %-10s %-10s %-30s %-10s\n" "${OS}" "${ARCH:-amd64}" "${RELEASE}" "${EDITION}" "${URL}"
exit 0
elif [ "${OPERATION}" == "test" ]; then
_record_test_result "${URL}"
exit 0
elif [ "${OPERATION}" == "download" ]; then
DIR="$(pwd)"
fi
if [ "${DIR}" != "$(pwd)" ] && ! mkdir -p "${DIR}" 2>/dev/null; then
echo "ERROR! Unable to create directory ${DIR}"
exit 1
fi
if [[ ${OS} != windows && ${OS} != macos && ${OS} != windows-server ]]; then
echo "Downloading ${PRETTY} ${RELEASE} ${EDITION}"
echo "- URL: ${URL}"
fi
if ! curl --progress-bar --location --output "${DIR}/${FILE}" --continue-at - --user-agent "${USER_AGENT}" "${HEADERS[@]}" -- "${URL}"; then
echo "ERROR! Failed to download ${URL} with curl."
rm -f "${DIR}/${FILE}"
fi
}
function zsync_get() {
local DIR="${2}"
local FILE="${1##*/}"
local OUT=""
local URL="${1}"
if [ "${OPERATION}" == "show" ]; then
printf "%-10s %-10s %-10s %-30s %-10s\n" "${OS}" "${ARCH:-amd64}" "${RELEASE}" "${EDITION}" "${URL}" | tee -a "${PUBLIC_DIR}/tmp_${OS}"
exit 0
elif [ "${OPERATION}" == "test" ]; then
_record_test_result "${URL}"
exit 0
elif command -v zsync &>/dev/null; then
if [ -n "${3}" ]; then
OUT="${3}"
else
OUT="${FILE}"
fi
if ! mkdir -p "${DIR}" 2>/dev/null; then
echo "ERROR! Unable to create directory ${DIR}"
exit 1
fi
echo "Downloading ${PRETTY} ${RELEASE} ${EDITION} from ${URL}"
# Only force http for zsync - not earlier because we might fall through here
if ! zsync "${URL/https/http}.zsync" -i "${DIR}/${OUT}" -o "${DIR}/${OUT}" 2>/dev/null; then
echo "ERROR! Failed to download ${URL/https/http}.zsync"
exit 1
fi
if [ -e "${DIR}/${OUT}.zs-old" ]; then
rm "${DIR}/${OUT}.zs-old"
fi
else
echo "INFO: zsync not found, falling back to curl"
if [ -n "${3}" ]; then
web_get "${1}" "${2}" "${3}"
else
web_get "${1}" "${2}"
fi
fi
}
function _write_todo_from_tmp() {
{
echo "### $(date '+%Y-%m-%d')"
printf "| %-12s | %-6s | %-10s | %-16s | %-40s | %-6s |\n" "OS" "Arch" "Release" "Edition" "URL" "Status"
printf "|-------------|------|-----------|------------------|----------------------------------------|--------|\n"
awk '
NF >= 5 && ($NF == "PASS" || $NF == "FAIL") {
os=$1; arch=$2; release=$3; check=$NF; url=$(NF-1)
edition=(NF >= 6) ? $4 : ""
printf "| %-12s | %-6s | %-10s | %-16s | %-40s | %-6s |\n", os, arch, release, edition, url, check
}
' "${PUBLIC_DIR}"/tmp_* 2>/dev/null | sort -u
echo
} >>TODO/all
}
function show_help() {
cat <<EOF
generate <flag> <argument>
Usage:
<OS> Generate current <OS> data
-t <OS> Test ISO links for <OS> (PASS/FAIL)
-a --all Generate all data
-t -a Test ISO links for all distros
-p --parallel Generate all data in parallel
-t -p Test ISO links for all distros in parallel
-h --help Show this usage message
-v --version Show version and exit
EOF
SUPPORTED=$(ls "$TEMPLATES_DIR/" | grep -v 'tmp_')
nOS=$(echo "$SUPPORTED" | wc -w)
echo "$nOS Available OS templates:"
echo ${SUPPORTED}
}
function _run_one() {
local os="${1}"
if [ ! -f "${TEMPLATES_DIR}/${os}" ]; then
echo "ERROR: OS \"${os}\" not found!"
exit 1
fi
_iterate_os "${os}"
write_output "${os}" || true
if [ "${OPERATION}" == "test" ]; then
# Skip table_write when running as a parallel worker — parent collects via _write_todo_from_tmp
[ -z "${_DH_PARALLEL_WORKER}" ] && table_write ./TODO/all
fi
}
function _run_parallel() {
rm -f TODO/all "${PUBLIC_DIR}"/*
touch TODO/all
local max_jobs
max_jobs="$(nproc 2>/dev/null || echo 4)"
[ "$max_jobs" -lt 1 ] && max_jobs=1
local test_flag=""
[ "${OPERATION}" == "test" ] && test_flag="-t"
# Collect templates list
local templates=() t
for t in "$TEMPLATES_DIR"/*; do
[ -f "$t" ] || continue
templates+=("$(basename "$t")")
done
local total="${#templates[@]}"
# All worker PIDs — refreshed on every spawn/reap; used by the trap.
local -a worker_pids=()
# shellcheck disable=SC2317
_dh_cleanup() {
local p
for p in "${worker_pids[@]}"; do
# Negative PID = whole process group (worker was started with setsid)
kill -TERM -- "-${p}" 2>/dev/null
done
sleep 1
for p in "${worker_pids[@]}"; do
kill -KILL -- "-${p}" 2>/dev/null
done
printf '\n[interrupted]\n' >&2
exit 130
}
trap _dh_cleanup INT TERM
# pid -> os mapping (parallel arrays) for progress reporting
local -a worker_os=()
local -a worker_status_files=()
local idx=0 done_count=0
while [ "${idx}" -lt "${total}" ] || [ "${#worker_pids[@]}" -gt 0 ]; do
# Spawn until pool is full or templates exhausted
while [ "${idx}" -lt "${total}" ] && [ "${#worker_pids[@]}" -lt "${max_jobs}" ]; do
local os_name="${templates[${idx}]}"
idx=$((idx + 1))
local status_file
status_file="$(mktemp -t "dh-status-${os_name}-XXXXXX")"
# setsid puts each worker in its own process group so the trap can
# nuke the whole subtree (curl, sub-shells) by killing -PGID.
setsid bash -c "
start=\$(date +%s)
status=done
'${SCRIPT_DIR}/generate' ${test_flag} '${os_name}' >/dev/null 2>&1 || status=FAIL
end=\$(date +%s)
printf '%s %ds\n' \"\$status\" \"\$((end - start))\" >'${status_file}'
" &
worker_pids+=("$!")
worker_os+=("${os_name}")
worker_status_files+=("${status_file}")
done
# Reap any finished workers (non-blocking)
local _alive_p=() _alive_o=() _alive_s=() i
for i in "${!worker_pids[@]}"; do
local p="${worker_pids[$i]}"
if kill -0 "$p" 2>/dev/null; then
_alive_p+=("$p")
_alive_o+=("${worker_os[$i]}")
_alive_s+=("${worker_status_files[$i]}")
else
wait "$p" 2>/dev/null
done_count=$((done_count + 1))
local _line
_line="$(cat "${worker_status_files[$i]}" 2>/dev/null)"
rm -f "${worker_status_files[$i]}"
printf '[%d/%d] %-25s %s\n' "${done_count}" "${total}" "${worker_os[$i]}" "${_line:-?}"
fi
done
worker_pids=("${_alive_p[@]}")
worker_os=("${_alive_o[@]}")
worker_status_files=("${_alive_s[@]}")
[ "${idx}" -ge "${total}" ] && [ "${#worker_pids[@]}" -eq 0 ] || sleep 0.2
done
trap - INT TERM
[ "${OPERATION}" == "test" ] && _write_todo_from_tmp
printf '\033[0m\033[?47l'
}
function _run_all() {
rm -f TODO/all "${PUBLIC_DIR}"/*
touch TODO/all
for OS_PATH in "$TEMPLATES_DIR"/*; do
OS=$(basename "${OS_PATH}")
_iterate_os "${OS}"
write_output "${OS}" || true
done
[ "${OPERATION}" == "test" ] && _write_todo_from_tmp
printf '\033[0m\033[?47l'
}
# ---------------------------------------------------------------------------
OPERATION='show'
_args=()
for _a in "$@"; do
case "$_a" in
-t | --test) OPERATION='test' ;;
*) _args+=("$_a") ;;
esac
done
set -- "${_args[@]}"
unset _args _a
table_clear
if [ -z "${1}" ]; then
show_help
exit 1
fi
case "${1}" in
'-p' | '--parallel') _run_parallel ;;
'-a' | '--all') _run_all ;;
'-h' | '--help') show_help ;;
'-v' | '--version') echo "$generateVersion" ;;
*) _run_one "${1}" ;;
esac
# vim:tabstop=4:shiftwidth=4:noexpandtab