#!/usr/bin/env bash # shellcheck disable=SC2317 # shellcheck source=lib.sh SCRIPT_DIR="$(dirname "$(realpath "$0")")" source "${SCRIPT_DIR}/lib.sh" export PUBLIC_DIR="${PUBLIC_DIR:-${SCRIPT_DIR}/public}" TEMPLATES_DIR="${TEMPLATES_DIR:-${SCRIPT_DIR}/templates}" generateVersion="0.6" check_hash() { local iso="" local hash="$2" local status="OK" if [ "${OPERATION}" == "download" ]; then iso="${1}" else iso="${VM_PATH}/${1}" fi local hash_algo case ${#hash} in 32) hash_algo=md5sum ;; 40) hash_algo=sha1sum ;; 64) hash_algo=sha256sum ;; 128) hash_algo=sha512sum ;; *) status="UNKNOWN" ;; esac if [[ "$status" != "UNKNOWN" ]]; then if ! echo "${hash} ${iso}" | ${hash_algo} --check --status; then status="FAIL" fi fi if [[ "$status" == "FAIL" ]]; then echo $"ERROR!" exit 1 fi } function write_output() { unset -f config_ editions_ arch_ get_ releases_ extract_ unset GUEST IMAGE_TYPE DW_ARCHITECTURE DW_CATEGORY DW_DESKTOP DW_URL ORIGIN EDITIONS . "${TEMPLATES_DIR}/${OS}" local DAT="${PUBLIC_DIR}/tmp_${OS}.dat" if [ ! -s "${DAT}" ]; then echo "WARNING: no entries collected for ${OS}, skipping" return 1 fi # In test mode, only PASS rows count; otherwise everything emitted is OK. local _filter='OK' [ "${OPERATION}" == "test" ] && _filter='PASS' # Collect releases/editions per arch from structured tmp_${OS}.dat declare -A _arch_releases=() _arch_editions=() local _all_archs="" _arch _rel _ed _url _st while IFS=$'\t' read -r _arch _rel _ed _url _st; do [[ "${_st}" == "${_filter}" ]] || continue [ -n "${_arch}" ] || continue [[ " ${_all_archs} " == *" ${_arch} "* ]] || _all_archs+=" ${_arch}" _arch_releases[${_arch}]+=" ${_rel}" [ "${_ed}" != "-" ] && [ -n "${_ed}" ] && _arch_editions[${_arch}]+=" ${_ed}" done < "${DAT}" _all_archs=$(printf '%s\n' ${_all_archs} | sort -u | grep -v '^$' | paste -sd ' ') [ -z "${_all_archs}" ] && { echo "WARNING: no PASS entries for ${OS}, skipping"; return 1; } # Default sets = union across archs local _default_rel _default_ed _all_rel="" _all_ed="" local _a for _a in ${_all_archs}; do _all_rel+=" ${_arch_releases[${_a}]}" _all_ed+=" ${_arch_editions[${_a}]}" done _default_rel=$(printf '%s\n' ${_all_rel} | sort -ur | grep -v '^$' | paste -sd ' ') _default_ed=$(printf '%s\n' ${_all_ed} | sort -u | grep -v '^$' | paste -sd ' ') # Per-arch overrides (only emitted when they differ from default) local _override_lines="" for _a in ${_all_archs}; do local _ar _ae _ar=$(printf '%s\n' ${_arch_releases[${_a}]} | sort -ur | grep -v '^$' | paste -sd ' ') _ae=$(printf '%s\n' ${_arch_editions[${_a}]} | sort -u | grep -v '^$' | paste -sd ' ') if [ "${_ar}" != "${_default_rel}" ]; then _override_lines+="RELEASES_${_a}=\"${_ar}\""$'\n' fi if [ -n "${_default_ed}" ] && [ "${_ae}" != "${_default_ed}" ]; then _override_lines+="EDITIONS_${_a}=\"${_ae}\""$'\n' fi done { printf '# Template file for '\''%s'\''\n' "$OS" printf 'OSNAME="%s"\n' "${OSNAME:-${OS}}" printf 'PRETTY="%s"\n' "${PRETTY:-}" printf 'LOGO="%s"\n' "${LOGO:-}" printf 'ICON="%s"\n' "${ICON:-}" printf 'ICON_ONLINE="%s"\n' "${ICON_ONLINE:-}" printf 'CATEGORY="%s"\n' "${CATEGORY:-}" printf 'BASEDOF="%s"\n' "${BASEDOF:-}" printf 'DESCRIPTION="%s"\n' "${DESCRIPTION:-}" printf 'HOMEPAGE="%s"\n' "${HOMEPAGE:-}" printf 'CREDENTIALS="%s"\n' "${CREDENTIALS:-}" printf 'GPG="%s"\n' "${GPG:-}" printf 'RSS="%s"\n' "${RSS:-}" printf 'DW="%s"\n' "${DW:-}" printf 'MAGNET="%s"\n' "${MAGNET:-}" printf 'CHAT="%s"\n' "${CHAT:-}" for _key in GUEST IMAGE_TYPE DW_ARCHITECTURE DW_CATEGORY DW_DESKTOP DW_URL ORIGIN; do local _val="${!_key}" [ -n "${_val}" ] && printf '%s="%s"\n' "${_key}" "${_val}" done echo printf 'RELEASES="%s"\n' "${_default_rel}" [ -n "${_default_ed}" ] && printf 'EDITIONS="%s"\n' "${_default_ed}" printf 'QEMU_ARCH="%s"\n' "${_all_archs}" if [ -n "${_override_lines}" ]; then echo printf '%s' "${_override_lines}" fi # Embed all template functions verbatim so public/ is self-contained. # get_entries_ is excluded — it's a build-time helper from lib.sh. while IFS= read -r _decl; do local _fn="${_decl#function }"; _fn="${_fn%%(*}" [[ "${_fn}" == "get_entries_" ]] && continue echo sed -n "/^${_decl}/,/^}/p" "${TEMPLATES_DIR}/${OS}" | sed '1s/^function //' done < <(grep "^function " "${TEMPLATES_DIR}/${OS}") } >"${PUBLIC_DIR}/${OS}" echo "wrote ${PUBLIC_DIR}/${OS}" } function is_arch_supported() { local OS="${1}" local CHECK_ARCH="${2}" local SUPPORTED="" if [[ $(type -t "arch_") == function ]]; then SUPPORTED=$(arch_) else SUPPORTED="amd64" fi [[ " ${SUPPORTED} " == *" ${CHECK_ARCH} "* ]] } function _iterate_os() { OS="${1}" mkdir -p "${PUBLIC_DIR}" rm -f "${PUBLIC_DIR}/${OS}" "${PUBLIC_DIR}/tmp_${OS}" "${PUBLIC_DIR}/tmp_${OS}.dat" touch "${PUBLIC_DIR}/tmp_${OS}" "${PUBLIC_DIR}/tmp_${OS}.dat" # Per-OS HTTP cache — exported so child shells / subshells see it. # Wiped on RETURN so each OS gets a fresh cache. export _WEB_CACHE_DIR _WEB_CACHE_DIR="$(mktemp -d -t "dh-webcache-${OS}-XXXXXX")" trap 'rm -rf "${_WEB_CACHE_DIR}"; unset _WEB_CACHE_DIR' RETURN unset -f get_ releases_ editions_ arch_ config_ extract_ . "${TEMPLATES_DIR}/${OS}" local ARCHS if [[ $(type -t "arch_") == function ]]; then ARCHS=$(arch_) else ARCHS="amd64" fi # Cache releases per arch once (releases_ may depend on ARCH/EDITION) for ARCH in ${ARCHS}; do for RELEASE in $(releases_); do . "${TEMPLATES_DIR}/${OS}" get_entries_ "${OS}" "${ARCH}" "${RELEASE}" "${OPERATION}" "${PUBLIC_DIR}" "${TEMPLATES_DIR}" done done } # Verify a checksum file against a GPG signature. # # Usage: verify_gpg SUMS_URL KEY_SOURCE # SUMS_URL — URL of the SHA256SUMS / MD5SUMS file already used for hash check # KEY_SOURCE — value of GPG="" from the template: # "" → skip (no GPG configured) # "FINGERPRINT" → fetch key from keyserver # "https://..." → download key from URL # # Signature URL is derived as SUMS_URL + ".asc" (fallback: ".gpg"). function verify_gpg() { local SUMS_URL="${1}" local KEY_SOURCE="${2:-${GPG:-}}" local SIG_FILE="" KEY_FILE="" SUMS_FILE="" GPG_HOME="" if [ -z "${KEY_SOURCE}" ]; then return 0 fi if ! command -v gpg &>/dev/null; then echo "WARNING: gpg not found — skipping signature verification" return 1 fi SIG_FILE=$(mktemp) SUMS_FILE=$(mktemp) GPG_HOME=$(mktemp -d) chmod 700 "${GPG_HOME}" # Download SUMS file curl --disable --silent --location -o "${SUMS_FILE}" "${SUMS_URL}" 2>/dev/null || { echo "WARNING: Failed to download SUMS file for GPG check — ${SUMS_URL}" rm -f "${SIG_FILE}" "${SUMS_FILE}"; rm -rf "${GPG_HOME}" return 1 } # Download signature (.asc preferred, fallback .gpg) if ! curl --disable --silent --location --fail -o "${SIG_FILE}" "${SUMS_URL}.asc" 2>/dev/null; then if ! curl --disable --silent --location --fail -o "${SIG_FILE}" "${SUMS_URL}.gpg" 2>/dev/null; then echo "WARNING: No GPG signature found at ${SUMS_URL}.asc or .gpg" rm -f "${SIG_FILE}" "${SUMS_FILE}"; rm -rf "${GPG_HOME}" return 1 fi fi # Import signing key KEY_FILE=$(mktemp) if [[ "${KEY_SOURCE}" =~ ^https?:// ]]; then curl --disable --silent --location -o "${KEY_FILE}" "${KEY_SOURCE}" 2>/dev/null || { echo "WARNING: Failed to download GPG key — ${KEY_SOURCE}" rm -f "${SIG_FILE}" "${SUMS_FILE}" "${KEY_FILE}"; rm -rf "${GPG_HOME}" return 1 } GNUPGHOME="${GPG_HOME}" gpg --quiet --import "${KEY_FILE}" 2>/dev/null else # Treat as fingerprint — fetch from keyserver GNUPGHOME="${GPG_HOME}" gpg --quiet --keyserver hkps://keyserver.ubuntu.com \ --recv-keys "${KEY_SOURCE}" 2>/dev/null || \ GNUPGHOME="${GPG_HOME}" gpg --quiet --keyserver hkps://keys.openpgp.org \ --recv-keys "${KEY_SOURCE}" 2>/dev/null fi local result=0 if GNUPGHOME="${GPG_HOME}" gpg --verify "${SIG_FILE}" "${SUMS_FILE}" 2>/dev/null; then echo "GPG OK — ${PRETTY:-${OS}} ${RELEASE}" else echo "WARNING: GPG verification FAILED — ${PRETTY:-${OS}} ${RELEASE}" result=1 fi rm -f "${SIG_FILE}" "${SUMS_FILE}" "${KEY_FILE}"; rm -rf "${GPG_HOME}" return ${result} } # Download a file from the web and pipe it to stdout function web_pipe() { local URL="${1}" local key body key=$(_web_cache_key GET "${URL}") if body=$(_web_cache_get_body "${key}"); then printf '%s' "${body}" return 0 fi body=$(curl --disable --silent --location "${URL}") printf '%s' "${body}" | _web_cache_put_body "${key}" printf '%s' "${body}" } # Download a JSON file from the web and pipe it to stdout function web_pipe_json() { local URL="${1}" local key body key=$(_web_cache_key GET_JSON "${URL}") if body=$(_web_cache_get_body "${key}"); then printf '%s' "${body}" return 0 fi body=$(curl --disable --silent --location --header "Accept: application/json" "${URL}") printf '%s' "${body}" | _web_cache_put_body "${key}" printf '%s' "${body}" } # checks if a URL is reachable function web_check() { local HEADERS=() local URL="${1}" # Process any headers while (("$#")); do if [ "${1}" == "--header" ]; then HEADERS+=("${1}" "${2}") shift 2 else shift fi done local key rc key=$(_web_cache_key "HEAD${HEADERS[*]}" "${URL}") if _web_cache_get_status "${key}"; then return 0; fi rc=$? # rc=2 means not cached; otherwise it's the cached exit status if [ "${rc}" != "2" ]; then return "${rc}"; fi curl --silent --location --head --output /dev/null --fail --connect-timeout 30 --max-time 30 --retry 3 "${HEADERS[@]}" "${URL}" rc=$? _web_cache_put_status "${key}" "${rc}" return "${rc}" } # checks if a URL needs to be redirected and returns the final URL function web_redirect() { local REDIRECT_URL="" local URL="${1}" # Use HEAD to follow redirects without downloading the body. REDIRECT_URL=$(curl --silent --location --head --max-time 10 --write-out '%{url_effective}' --output /dev/null "${URL}" 2>/dev/null) if [ -n "${REDIRECT_URL}" ] && [ "${REDIRECT_URL}" != "${URL}" ]; then echo "${REDIRECT_URL}" else echo "${URL}" fi } function _record_test_result() { local URL="${1}" local CHECK CHECK=$(web_check "${URL}" && echo "PASS" || echo "FAIL") table_add_row "$OS" "${ARCH:-amd64}" "$RELEASE" "$EDITION" "$URL" "$CHECK" printf "%-10s %-15s %-15s %-10s %-30s %-10s\n" "${OS}" "${ARCH:-amd64}" "${RELEASE}" "${EDITION}" "${URL}" "${CHECK}" | tee -a "${PUBLIC_DIR}/tmp_${OS}" } # Download a file from the web function web_get() { local HEADERS=() local URL="${1}" local DIR="${2}" local FILE="" local USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36" if [ -n "${3}" ]; then FILE="${3}" else FILE="${URL##*/}" fi # Process any URL redirections after the file name has been extracted URL=$(web_redirect "${URL}") # Process any headers while (("$#")); do if [ "${1}" == "--header" ]; then HEADERS+=("${1}" "${2}") shift 2 else shift fi done if [ "${OPERATION}" == "show" ]; then printf "%-10s %-10s %-10s %-30s %-10s\n" "${OS}" "${ARCH:-amd64}" "${RELEASE}" "${EDITION}" "${URL}" exit 0 elif [ "${OPERATION}" == "test" ]; then _record_test_result "${URL}" exit 0 elif [ "${OPERATION}" == "download" ]; then DIR="$(pwd)" fi if [ "${DIR}" != "$(pwd)" ] && ! mkdir -p "${DIR}" 2>/dev/null; then echo "ERROR! Unable to create directory ${DIR}" exit 1 fi if [[ ${OS} != windows && ${OS} != macos && ${OS} != windows-server ]]; then echo "Downloading ${PRETTY} ${RELEASE} ${EDITION}" echo "- URL: ${URL}" fi if ! curl --progress-bar --location --output "${DIR}/${FILE}" --continue-at - --user-agent "${USER_AGENT}" "${HEADERS[@]}" -- "${URL}"; then echo "ERROR! Failed to download ${URL} with curl." rm -f "${DIR}/${FILE}" fi } function zsync_get() { local DIR="${2}" local FILE="${1##*/}" local OUT="" local URL="${1}" if [ "${OPERATION}" == "show" ]; then printf "%-10s %-10s %-10s %-30s %-10s\n" "${OS}" "${ARCH:-amd64}" "${RELEASE}" "${EDITION}" "${URL}" | tee -a "${PUBLIC_DIR}/tmp_${OS}" exit 0 elif [ "${OPERATION}" == "test" ]; then _record_test_result "${URL}" exit 0 elif command -v zsync &>/dev/null; then if [ -n "${3}" ]; then OUT="${3}" else OUT="${FILE}" fi if ! mkdir -p "${DIR}" 2>/dev/null; then echo "ERROR! Unable to create directory ${DIR}" exit 1 fi echo "Downloading ${PRETTY} ${RELEASE} ${EDITION} from ${URL}" # Only force http for zsync - not earlier because we might fall through here if ! zsync "${URL/https/http}.zsync" -i "${DIR}/${OUT}" -o "${DIR}/${OUT}" 2>/dev/null; then echo "ERROR! Failed to download ${URL/https/http}.zsync" exit 1 fi if [ -e "${DIR}/${OUT}.zs-old" ]; then rm "${DIR}/${OUT}.zs-old" fi else echo "INFO: zsync not found, falling back to curl" if [ -n "${3}" ]; then web_get "${1}" "${2}" "${3}" else web_get "${1}" "${2}" fi fi } function _write_todo_from_tmp() { { echo "### $(date '+%Y-%m-%d')" printf "| %-12s | %-6s | %-10s | %-16s | %-40s | %-6s |\n" "OS" "Arch" "Release" "Edition" "URL" "Status" printf "|-------------|------|-----------|------------------|----------------------------------------|--------|\n" awk ' NF >= 5 && ($NF == "PASS" || $NF == "FAIL") { os=$1; arch=$2; release=$3; check=$NF; url=$(NF-1) edition=(NF >= 6) ? $4 : "" printf "| %-12s | %-6s | %-10s | %-16s | %-40s | %-6s |\n", os, arch, release, edition, url, check } ' "${PUBLIC_DIR}"/tmp_* 2>/dev/null | sort -u echo } >>TODO/all } function show_help() { cat < Usage: Generate current data -t Test ISO links for (PASS/FAIL) -a --all Generate all data -t -a Test ISO links for all distros -p --parallel Generate all data in parallel -t -p Test ISO links for all distros in parallel -h --help Show this usage message -v --version Show version and exit EOF SUPPORTED=$(ls "$TEMPLATES_DIR/" | grep -v 'tmp_') nOS=$(echo "$SUPPORTED" | wc -w) echo "$nOS Available OS templates:" echo ${SUPPORTED} } function _run_one() { local os="${1}" if [ ! -f "${TEMPLATES_DIR}/${os}" ]; then echo "ERROR: OS \"${os}\" not found!" exit 1 fi _iterate_os "${os}" write_output "${os}" || true if [ "${OPERATION}" == "test" ]; then # Skip table_write when running as a parallel worker — parent collects via _write_todo_from_tmp [ -z "${_DH_PARALLEL_WORKER}" ] && table_write ./TODO/all fi } function _run_parallel() { rm -f TODO/all "${PUBLIC_DIR}"/* touch TODO/all local max_jobs max_jobs="$(nproc 2>/dev/null || echo 4)" [ "$max_jobs" -lt 1 ] && max_jobs=1 local test_flag="" [ "${OPERATION}" == "test" ] && test_flag="-t" # Collect templates list local templates=() t for t in "$TEMPLATES_DIR"/*; do [ -f "$t" ] || continue templates+=("$(basename "$t")") done local total="${#templates[@]}" # All worker PIDs — refreshed on every spawn/reap; used by the trap. local -a worker_pids=() # shellcheck disable=SC2317 _dh_cleanup() { local p for p in "${worker_pids[@]}"; do # Negative PID = whole process group (worker was started with setsid) kill -TERM -- "-${p}" 2>/dev/null done sleep 1 for p in "${worker_pids[@]}"; do kill -KILL -- "-${p}" 2>/dev/null done printf '\n[interrupted]\n' >&2 exit 130 } trap _dh_cleanup INT TERM # pid -> os mapping (parallel arrays) for progress reporting local -a worker_os=() local -a worker_status_files=() local idx=0 done_count=0 while [ "${idx}" -lt "${total}" ] || [ "${#worker_pids[@]}" -gt 0 ]; do # Spawn until pool is full or templates exhausted while [ "${idx}" -lt "${total}" ] && [ "${#worker_pids[@]}" -lt "${max_jobs}" ]; do local os_name="${templates[${idx}]}" idx=$((idx + 1)) local status_file status_file="$(mktemp -t "dh-status-${os_name}-XXXXXX")" # setsid puts each worker in its own process group so the trap can # nuke the whole subtree (curl, sub-shells) by killing -PGID. setsid bash -c " start=\$(date +%s) status=done '${SCRIPT_DIR}/generate' ${test_flag} '${os_name}' >/dev/null 2>&1 || status=FAIL end=\$(date +%s) printf '%s %ds\n' \"\$status\" \"\$((end - start))\" >'${status_file}' " & worker_pids+=("$!") worker_os+=("${os_name}") worker_status_files+=("${status_file}") done # Reap any finished workers (non-blocking) local _alive_p=() _alive_o=() _alive_s=() i for i in "${!worker_pids[@]}"; do local p="${worker_pids[$i]}" if kill -0 "$p" 2>/dev/null; then _alive_p+=("$p") _alive_o+=("${worker_os[$i]}") _alive_s+=("${worker_status_files[$i]}") else wait "$p" 2>/dev/null done_count=$((done_count + 1)) local _line _line="$(cat "${worker_status_files[$i]}" 2>/dev/null)" rm -f "${worker_status_files[$i]}" printf '[%d/%d] %-25s %s\n' "${done_count}" "${total}" "${worker_os[$i]}" "${_line:-?}" fi done worker_pids=("${_alive_p[@]}") worker_os=("${_alive_o[@]}") worker_status_files=("${_alive_s[@]}") [ "${idx}" -ge "${total}" ] && [ "${#worker_pids[@]}" -eq 0 ] || sleep 0.2 done trap - INT TERM [ "${OPERATION}" == "test" ] && _write_todo_from_tmp printf '\033[0m\033[?47l' } function _run_all() { rm -f TODO/all "${PUBLIC_DIR}"/* touch TODO/all for OS_PATH in "$TEMPLATES_DIR"/*; do OS=$(basename "${OS_PATH}") _iterate_os "${OS}" write_output "${OS}" || true done [ "${OPERATION}" == "test" ] && _write_todo_from_tmp printf '\033[0m\033[?47l' } # --------------------------------------------------------------------------- OPERATION='show' _args=() for _a in "$@"; do case "$_a" in -t | --test) OPERATION='test' ;; *) _args+=("$_a") ;; esac done set -- "${_args[@]}" unset _args _a table_clear if [ -z "${1}" ]; then show_help exit 1 fi case "${1}" in '-p' | '--parallel') _run_parallel ;; '-a' | '--all') _run_all ;; '-h' | '--help') show_help ;; '-v' | '--version') echo "$generateVersion" ;; *) _run_one "${1}" ;; esac # vim:tabstop=4:shiftwidth=4:noexpandtab