From 7586aea4cda4c4a900288a90efbd0874c5eda6fe Mon Sep 17 00:00:00 2001 From: MickLesk Date: Wed, 27 May 2026 13:49:16 +0200 Subject: [PATCH] add tools.func from upstream --- scripts/core/tools.func | 2664 +++++++++++++++++++++++++------------ src/server/ssh-service.js | 82 +- 2 files changed, 1870 insertions(+), 876 deletions(-) diff --git a/scripts/core/tools.func b/scripts/core/tools.func index eb64a9a..9e600f3 100644 --- a/scripts/core/tools.func +++ b/scripts/core/tools.func @@ -477,6 +477,21 @@ install_packages_with_retry() { done msg_error "Failed to install packages after $((max_retries + 1)) attempts: ${packages[*]}" + # Provide a quick diagnostic: check if package exists in any configured repo + local _os_codename + _os_codename=$(get_os_info codename) + local _unavailable=() + for _pkg in "${packages[@]}"; do + if ! apt-cache show "$_pkg" &>/dev/null; then + _unavailable+=("$_pkg") + fi + done + if [[ ${#_unavailable[@]} -gt 0 ]]; then + msg_error "Package(s) not found in any configured repository: ${_unavailable[*]}" + msg_error "Hint: These packages may not be available for '${_os_codename}'. Check repository configuration or package names." + else + msg_error "Hint: Package(s) exist in the repo but could not be installed — run 'apt-get install -f' inside the container or check for dependency conflicts." + fi return 100 } @@ -508,6 +523,7 @@ upgrade_packages_with_retry() { done msg_error "Failed to upgrade packages after $((max_retries + 1)) attempts: ${packages[*]}" + msg_error "Hint: The package may be held back, have conflicting dependencies, or the repository is unreachable. Check 'apt-cache policy ${packages[*]}' inside the container." return 100 } @@ -700,7 +716,7 @@ manage_tool_repository() { local gpg_key_url="${4:-}" local distro_id repo_component suite - distro_id=$(awk -F= '/^ID=/{print $2}' /etc/os-release | tr -d '"') + distro_id=$(get_os_info id) case "$tool_name" in mariadb) @@ -714,7 +730,7 @@ manage_tool_repository() { # Get suite for fallback handling local distro_codename - distro_codename=$(awk -F= '/^VERSION_CODENAME=/{print $2}' /etc/os-release) + distro_codename=$(get_os_info codename) suite=$(get_fallback_suite "$distro_id" "$distro_codename" "$repo_url/$distro_id") # Setup new repository using deb822 format @@ -739,13 +755,14 @@ manage_tool_repository() { # Import GPG key with retry logic if ! download_gpg_key "$gpg_key_url" "/etc/apt/keyrings/mongodb-server-${version}.gpg" "dearmor"; then msg_error "Failed to download MongoDB GPG key" + msg_error "Hint: Check connectivity to downloads.mongodb.org or verify MongoDB version ${version} is still available" return 7 fi chmod 644 "/etc/apt/keyrings/mongodb-server-${version}.gpg" # Setup repository local distro_codename - distro_codename=$(awk -F= '/^VERSION_CODENAME=/{print $2}' /etc/os-release) + distro_codename=$(get_os_info codename) # Suite mapping with fallback for newer releases not yet supported by upstream if [[ "$distro_id" == "debian" ]]; then @@ -816,7 +833,7 @@ EOF # NodeSource uses deb822 format with GPG from repo local distro_codename - distro_codename=$(awk -F= '/^VERSION_CODENAME=/{print $2}' /etc/os-release) + distro_codename=$(get_os_info codename) # Download GPG key from NodeSource with retry logic if ! download_gpg_key "$gpg_key_url" "/etc/apt/keyrings/nodesource.gpg" "dearmor"; then @@ -846,6 +863,7 @@ EOF # Download and install keyring with retry logic if ! curl_with_retry "$gpg_key_url" "/tmp/debsuryorg-archive-keyring.deb"; then msg_error "Failed to download PHP keyring" + msg_error "Hint: Check connectivity to packages.sury.org (packages.sury.org/php)" return 7 fi # Don't use /dev/null redirection for dpkg as it may use background processes @@ -858,7 +876,7 @@ EOF # Setup repository local distro_codename - distro_codename=$(awk -F= '/^VERSION_CODENAME=/{print $2}' /etc/os-release) + distro_codename=$(get_os_info codename) cat </etc/apt/sources.list.d/php.sources Types: deb URIs: https://packages.sury.org/php @@ -886,7 +904,7 @@ EOF # Setup repository local distro_codename - distro_codename=$(awk -F= '/^VERSION_CODENAME=/{print $2}' /etc/os-release) + distro_codename=$(get_os_info codename) cat </etc/apt/sources.list.d/postgresql.sources Types: deb URIs: http://apt.postgresql.org/pub/repos/apt @@ -1468,6 +1486,7 @@ download_file() { done msg_error "Failed to download: $url" + msg_error "Hint: Check network connectivity or DNS resolution. The server may be unreachable or the URL may have changed." return 250 } @@ -1730,6 +1749,7 @@ wait_for_apt() { while is_apt_locked; do if [[ $waited -ge $max_wait ]]; then msg_error "Timeout waiting for apt to be available" + msg_error "Hint: Another process (apt, dpkg, unattended-upgrades) may hold a lock. Check: ps aux | grep -E 'apt|dpkg'" return 100 fi @@ -1896,7 +1916,8 @@ setup_deb822_repo() { local tmp_gpg tmp_gpg=$(mktemp) || return 252 curl -fsSL "$gpg_url" -o "$tmp_gpg" || { - msg_error "Failed to download GPG key for ${name}" + msg_error "Failed to download GPG key for ${name} from: ${gpg_url}" + msg_error "Hint: Check network connectivity. If behind a proxy or firewall, ensure HTTPS access to $(echo "$gpg_url" | grep -oE 'https?://[^/]+') is allowed." rm -f "$tmp_gpg" return 7 } @@ -1924,8 +1945,8 @@ setup_deb822_repo() { echo "Types: deb" echo "URIs: $repo_url" echo "Suites: $suite" - # Flat repositories (suite="./" or absolute path) must not have Components - if [[ "$suite" != "./" && -n "$component" ]]; then + # Flat repositories (suite ending with "/" or "./") must not have Components + if [[ "$suite" != *"/" && -n "$component" ]]; then echo "Components: $component" fi [[ -n "$architectures" ]] && echo "Architectures: $architectures" @@ -2079,15 +2100,33 @@ get_latest_gh_tag() { local temp_file temp_file=$(mktemp) - if ! github_api_call "https://api.github.com/repos/${repo}/tags?per_page=50" "$temp_file"; then - rm -f "$temp_file" - return 22 - fi - local tag="" + if [[ -n "$prefix" ]]; then - tag=$(jq -r --arg p "$prefix" '[.[] | select(.name | startswith($p))][0].name // empty' "$temp_file") + # Use git/matching-refs API for server-side prefix filtering. This avoids + # paging through unrelated tags (e.g. mongodb/mongo-tools where 100.x tags + # only appear after page 4 of /tags). Returns ALL tags matching the prefix + # in a single call, sorted lexicographically ascending; we pick the + # highest version using `sort -V`. + if ! github_api_call "https://api.github.com/repos/${repo}/git/matching-refs/tags/${prefix}" "$temp_file"; then + rm -f "$temp_file" + return 22 + fi + + local count + count=$(jq 'length' "$temp_file" 2>/dev/null || echo 0) + if [[ "$count" -gt 0 ]]; then + tag=$(jq -r '.[].ref' "$temp_file" | + sed 's|^refs/tags/||' | + sort -V | + tail -n1) + fi else + # No prefix: just take the first (newest) tag from /tags + if ! github_api_call "https://api.github.com/repos/${repo}/tags?per_page=1" "$temp_file"; then + rm -f "$temp_file" + return 22 + fi tag=$(jq -r '.[0].name // empty' "$temp_file") fi @@ -2376,11 +2415,12 @@ check_for_gh_release() { # For pinned versions, query the specific release tag directly if [[ -n "$pinned_version_in" ]]; then + local pinned_version_encoded="${pinned_version_in//\//%2F}" http_code=$(curl -sSL --max-time 20 -w "%{http_code}" -o /tmp/gh_check.json \ -H 'Accept: application/vnd.github+json' \ -H 'X-GitHub-Api-Version: 2022-11-28' \ "${header_args[@]}" \ - "https://api.github.com/repos/${source}/releases/tags/${pinned_version_in}" 2>/dev/null) || true + "https://api.github.com/repos/${source}/releases/tags/${pinned_version_encoded}" 2>/dev/null) || true if [[ "$http_code" == "200" ]] && [[ -s /tmp/gh_check.json ]]; then releases_json="[$(/dev/null; then kill "$SPINNER_PID" >/dev/null; fi @@ -2751,7 +2797,7 @@ function download_with_progress() { # - Adds to /root/.bashrc for non-login shells (pct enter) # ------------------------------------------------------------------------------ -function ensure_usr_local_bin_persist() { +ensure_usr_local_bin_persist() { # Skip on Proxmox host command -v pveversion &>/dev/null && return @@ -2770,25 +2816,34 @@ function ensure_usr_local_bin_persist() { } # ------------------------------------------------------------------------------ -# curl_download - Downloads a file with automatic retry and exponential backoff. +# curl_download - Downloads a file with stall detection and retry. # # Usage: curl_download # -# Retries up to 5 times with increasing --max-time (60/120/240/480/960s). -# Returns 0 on success, 1 if all attempts fail. +# Uses --speed-limit / --speed-time instead of a hard --max-time cap so that +# slow but progressing downloads (e.g. large .deb files from slow mirrors) are +# never aborted mid-transfer. Only aborts when throughput drops below 1 KB/s +# for 60 consecutive seconds (i.e. a genuine stall or dead connection). +# Retries up to 3 times on failure. +# Returns 0 on success, 7 if all attempts fail. # ------------------------------------------------------------------------------ -function curl_download() { +curl_download() { local output="$1" local url="$2" - local timeouts=(60 120 240 480 960) + local retries=3 + local attempt=1 - for i in "${!timeouts[@]}"; do - if curl --connect-timeout 15 --max-time "${timeouts[$i]}" -fsSL -o "$output" "$url"; then + while ((attempt <= retries)); do + if curl --connect-timeout 15 \ + --speed-limit 1024 \ + --speed-time 60 \ + -fsSL -o "$output" "$url"; then return 0 fi - if ((i < ${#timeouts[@]} - 1)); then - msg_warn "Download timed out after ${timeouts[$i]}s, retrying... (attempt $((i + 2))/${#timeouts[@]})" + if ((attempt < retries)); then + msg_warn "Download failed or stalled (attempt ${attempt}/${retries}), retrying..." fi + ((attempt++)) done return 7 } @@ -2839,7 +2894,67 @@ function curl_download() { # fetch_and_deploy_codeberg_release "autocaliweb" "gelbphoenix/autocaliweb" "tag" "v0.11.3" "/opt/autocaliweb" # ------------------------------------------------------------------------------ -function fetch_and_deploy_codeberg_release() { +# ------------------------------------------------------------------------------ +# _diagnose_deb_failure() +# +# - Called when both apt and dpkg fail to install a .deb package +# - Extracts package metadata and detects common failure patterns +# - Outputs enhanced error messages with actionable hints: +# * PostgreSQL version conflicts (e.g., postgresql-16-foo with pg17 active) +# * Missing declared dependencies +# * Generic fallback hint pointing to the log +# +# Usage: _diagnose_deb_failure "/path/to/file.deb" +# Returns: always 0 (diagnostic only — caller must return the error code) +# ------------------------------------------------------------------------------ +_diagnose_deb_failure() { + local deb_path="$1" + local filename="${deb_path##*/}" + local pkg_name pkg_deps pkg_version + + pkg_name=$(dpkg-deb -f "$deb_path" Package 2>/dev/null || echo "${filename%%_*}") + pkg_version=$(dpkg-deb -f "$deb_path" Version 2>/dev/null || true) + pkg_deps=$(dpkg-deb -f "$deb_path" Depends 2>/dev/null || true) + + msg_error "Failed to install '${pkg_name}${pkg_version:+ (${pkg_version})}' — both apt and dpkg reported errors" + + # Detect PostgreSQL version conflict (e.g., postgresql-16-vchord while pg17 is active) + local pg_ver_needed pg_ver_installed + pg_ver_needed=$(echo "$filename" | grep -oP '(?<=postgresql-)[0-9]+(?=-)' | head -1 || true) + if [[ -n "$pg_ver_needed" ]]; then + pg_ver_installed=$(psql -V 2>/dev/null | awk '{print $3}' | cut -d. -f1 || true) + if [[ -n "$pg_ver_installed" && "$pg_ver_needed" != "$pg_ver_installed" ]]; then + msg_error "Version conflict: '${pkg_name}' is built for PostgreSQL ${pg_ver_needed}, but PostgreSQL ${pg_ver_installed} is installed on this system." + msg_error "Hint: Your distribution installed a different PostgreSQL version than expected. The script may need updating to use postgresql-${pg_ver_installed}-* packages." + return 0 + fi + fi + + # Show which declared dependencies are not satisfied + if [[ -n "$pkg_deps" ]]; then + local missing_deps=() + while IFS=',' read -ra dep_list; do + for dep_entry in "${dep_list[@]}"; do + local dep_pkg + dep_pkg=$(echo "$dep_entry" | awk '{print $1}' | tr -d ' ') + [[ -z "$dep_pkg" || "$dep_pkg" == "("* ]] && continue + if ! dpkg-query -W -f='${Status}' "$dep_pkg" 2>/dev/null | grep -q "install ok installed"; then + missing_deps+=("$dep_pkg") + fi + done + done <<<"$pkg_deps" + if [[ ${#missing_deps[@]} -gt 0 ]]; then + msg_error "Unmet dependencies: ${missing_deps[*]}" + msg_error "Hint: Run 'apt-get install -f' inside the container to attempt automatic dependency resolution." + else + msg_error "Hint: Declared dependencies appear present but installation still failed. Check the log above for the exact error." + fi + else + msg_error "Hint: Check the installation log above for the exact dependency or configuration error." + fi +} + +fetch_and_deploy_codeberg_release() { local app="$1" local repo="$2" local mode="${3:-tarball}" # tarball | binary | prebuild | singlefile | tag @@ -3072,7 +3187,7 @@ function fetch_and_deploy_codeberg_release() { chmod 644 "$tmpdir/$filename" $STD apt install -y "$tmpdir/$filename" || { $STD dpkg -i "$tmpdir/$filename" || { - msg_error "Both apt and dpkg installation failed" + _diagnose_deb_failure "$tmpdir/$filename" rm -rf "$tmpdir" return 100 } @@ -3381,7 +3496,7 @@ _gh_scan_older_releases() { return 250 } -function fetch_and_deploy_gh_release() { +fetch_and_deploy_gh_release() { local app="$1" local repo="$2" local mode="${3:-tarball}" # tarball | binary | prebuild | singlefile @@ -3617,7 +3732,7 @@ function fetch_and_deploy_gh_release() { [[ "${DPKG_FORCE_CONFNEW:-}" == "1" ]] && dpkg_opts="-o Dpkg::Options::=--force-confnew" DEBIAN_FRONTEND=noninteractive SYSTEMD_OFFLINE=1 $STD apt install -y $dpkg_opts "$tmpdir/$filename" || { SYSTEMD_OFFLINE=1 $STD dpkg -i "$tmpdir/$filename" || { - msg_error "Both apt and dpkg installation failed" + _diagnose_deb_failure "$tmpdir/$filename" rm -rf "$tmpdir" return 100 } @@ -3820,12 +3935,13 @@ function fetch_and_deploy_gh_release() { # - Supports Alpine and Debian-based systems # ------------------------------------------------------------------------------ -function setup_adminer() { +setup_adminer() { if grep -qi alpine /etc/os-release; then msg_info "Setup Adminer (Alpine)" mkdir -p /var/www/localhost/htdocs/adminer if ! curl_with_retry "https://github.com/vrana/adminer/releases/latest/download/adminer.php" "/var/www/localhost/htdocs/adminer/index.php"; then msg_error "Failed to download Adminer" + msg_error "Hint: Check connectivity to github.com/vrana/adminer (GitHub Releases)" return 250 fi cache_installed_version "adminer" "latest-alpine" @@ -3848,6 +3964,125 @@ function setup_adminer() { fi } +# ------------------------------------------------------------------------------ +# Installs or upgrades ClickHouse database server. +# +# Description: +# - Adds ClickHouse official repository +# - Installs specified version +# - Configures systemd service +# - Supports Debian/Ubuntu with fallback mechanism +# +# Variables: +# CLICKHOUSE_VERSION - ClickHouse version to install (default: latest) +# ------------------------------------------------------------------------------ + +setup_clickhouse() { + local CLICKHOUSE_VERSION="${CLICKHOUSE_VERSION:-latest}" + local DISTRO_ID DISTRO_CODENAME + DISTRO_ID=$(get_os_info id) + DISTRO_CODENAME=$(get_os_info codename) + + # Ensure non-interactive mode for all apt operations + export DEBIAN_FRONTEND=noninteractive + export NEEDRESTART_MODE=a + export NEEDRESTART_SUSPEND=1 + + # Resolve "latest" version + if [[ "$CLICKHOUSE_VERSION" == "latest" ]]; then + CLICKHOUSE_VERSION=$(curl -fsSL --max-time 15 https://packages.clickhouse.com/tgz/stable/ 2>/dev/null | + grep -oP 'clickhouse-common-static-\K[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | + sort -V | tail -n1 || echo "") + + # Fallback to GitHub API if package server failed + if [[ -z "$CLICKHOUSE_VERSION" ]]; then + CLICKHOUSE_VERSION=$(get_latest_github_release "ClickHouse/ClickHouse") || true + fi + + [[ -z "$CLICKHOUSE_VERSION" ]] && { + msg_error "Could not determine latest ClickHouse version from any source" + return 250 + } + fi + + # Get currently installed version + local CURRENT_VERSION="" + if command -v clickhouse-server >/dev/null 2>&1; then + CURRENT_VERSION=$(clickhouse-server --version 2>/dev/null | grep -oP 'version \K[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n1) + fi + + # Scenario 1: Already at target version - just update packages + if [[ -n "$CURRENT_VERSION" && "$CURRENT_VERSION" == "$CLICKHOUSE_VERSION" ]]; then + msg_info "Update ClickHouse $CLICKHOUSE_VERSION" + ensure_apt_working || return 100 + + # Perform upgrade with retry logic (non-fatal if fails) + upgrade_packages_with_retry "clickhouse-server" "clickhouse-client" || { + msg_warn "ClickHouse package upgrade had issues, continuing with current version" + } + cache_installed_version "clickhouse" "$CLICKHOUSE_VERSION" + msg_ok "Update ClickHouse $CLICKHOUSE_VERSION" + return 0 + fi + + # Scenario 2: Different version - clean upgrade + if [[ -n "$CURRENT_VERSION" && "$CURRENT_VERSION" != "$CLICKHOUSE_VERSION" ]]; then + msg_info "Upgrade ClickHouse from $CURRENT_VERSION to $CLICKHOUSE_VERSION" + stop_all_services "clickhouse-server" + remove_old_tool_version "clickhouse" + else + msg_info "Setup ClickHouse $CLICKHOUSE_VERSION" + fi + + ensure_dependencies apt-transport-https ca-certificates dirmngr gnupg + + # Prepare repository (cleanup + validation) + prepare_repository_setup "clickhouse" || { + msg_error "Failed to prepare ClickHouse repository" + return 100 + } + + # Setup repository (ClickHouse uses 'stable' suite) + setup_deb822_repo \ + "clickhouse" \ + "https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key" \ + "https://packages.clickhouse.com/deb" \ + "stable" \ + "main" + + # Install packages with retry logic + $STD apt update || { + msg_error "APT update failed for ClickHouse repository" + return 100 + } + + install_packages_with_retry "clickhouse-server" "clickhouse-client" || { + msg_error "Failed to install ClickHouse packages" + return 100 + } + + # Verify installation + if ! command -v clickhouse-server >/dev/null 2>&1; then + msg_error "ClickHouse installation completed but clickhouse-server command not found" + return 127 + fi + + # Setup data directory + mkdir -p /var/lib/clickhouse + if id clickhouse >/dev/null 2>&1; then + chown -R clickhouse:clickhouse /var/lib/clickhouse + fi + + # Enable and start service + $STD systemctl enable clickhouse-server || { + msg_warn "Failed to enable clickhouse-server service" + } + safe_service_restart clickhouse-server || true + + cache_installed_version "clickhouse" "$CLICKHOUSE_VERSION" + msg_ok "Setup ClickHouse $CLICKHOUSE_VERSION" +} + # ------------------------------------------------------------------------------ # Installs or updates Composer globally (robust, idempotent). # @@ -3857,7 +4092,7 @@ function setup_adminer() { # - Auto-updates to latest version # ------------------------------------------------------------------------------ -function setup_composer() { +setup_composer() { local COMPOSER_BIN="/usr/local/bin/composer" export COMPOSER_ALLOW_SUPERUSER=1 @@ -3892,6 +4127,7 @@ function setup_composer() { if ! curl_with_retry "https://getcomposer.org/installer" "/tmp/composer-setup.php"; then msg_error "Failed to download Composer installer" + msg_error "Hint: Check connectivity to getcomposer.org" return 250 fi @@ -3918,6 +4154,275 @@ function setup_composer() { msg_ok "Setup Composer" } +# ------------------------------------------------------------------------------ +# Docker Engine Installation and Management (All-In-One) +# +# Description: +# - By default uses distro repository (docker.io) for stability +# - Optionally uses official Docker repository for latest features +# - Detects and migrates old Docker installations +# - Optional: Installs/Updates Portainer CE +# - Updates running containers interactively +# - Cleans up legacy repository files +# +# Usage: +# setup_docker # Uses distro package (recommended) +# USE_DOCKER_REPO=true setup_docker # Uses official Docker repo +# DOCKER_PORTAINER="true" setup_docker +# DOCKER_LOG_DRIVER="json-file" setup_docker +# +# Variables: +# USE_DOCKER_REPO - Set to "true" to use official Docker repository +# (default: false, uses distro docker.io package) +# DOCKER_PORTAINER - Install Portainer CE (optional, "true" to enable) +# DOCKER_LOG_DRIVER - Log driver (optional, default: "journald") +# DOCKER_SKIP_UPDATES - Skip container update check (optional, "true" to skip) +# +# Features: +# - Uses stable distro packages by default +# - Migrates from get.docker.com to repository-based installation +# - Updates Docker Engine if newer version available +# - Interactive container update with multi-select +# - Portainer installation and update support +# ------------------------------------------------------------------------------ +setup_docker() { + local docker_installed=false + local portainer_installed=false + local USE_DOCKER_REPO="${USE_DOCKER_REPO:-false}" + + # Check if Docker is already installed + if command -v docker &>/dev/null; then + docker_installed=true + DOCKER_CURRENT_VERSION=$(docker --version | grep -oP '\d+\.\d+\.\d+' | head -1) + msg_info "Docker $DOCKER_CURRENT_VERSION detected" + fi + + # Check if Portainer is running + if docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^portainer$'; then + portainer_installed=true + msg_info "Portainer container detected" + fi + + # Scenario 1: Use distro repository (default, most stable) + if [[ "$USE_DOCKER_REPO" != "true" && "$USE_DOCKER_REPO" != "TRUE" && "$USE_DOCKER_REPO" != "1" ]]; then + + # Install or upgrade Docker from distro repo + if [ "$docker_installed" = true ]; then + msg_info "Checking for Docker updates (distro package)" + ensure_apt_working || return 100 + upgrade_packages_with_retry "docker.io" "docker-compose" || true + DOCKER_CURRENT_VERSION=$(docker --version | grep -oP '\d+\.\d+\.\d+' | head -1) + msg_ok "Docker is up-to-date ($DOCKER_CURRENT_VERSION)" + else + msg_info "Installing Docker (distro package)" + ensure_apt_working || return 100 + + # Install docker.io and docker-compose from distro + if ! install_packages_with_retry "docker.io"; then + msg_error "Failed to install docker.io from distro repository" + return 100 + fi + # docker-compose is optional + $STD apt install -y docker-compose 2>/dev/null || + msg_warn "Optional docker-compose not available from distro repository — use 'docker compose' plugin instead if needed" + + DOCKER_CURRENT_VERSION=$(docker --version | grep -oP '\d+\.\d+\.\d+' | head -1) + msg_ok "Installed Docker $DOCKER_CURRENT_VERSION (distro package)" + fi + + # Configure daemon.json + local log_driver="${DOCKER_LOG_DRIVER:-journald}" + mkdir -p /etc/docker + if [ ! -f /etc/docker/daemon.json ]; then + cat </etc/docker/daemon.json +{ + "log-driver": "$log_driver" +} +EOF + fi + + # Enable and start Docker + systemctl enable -q --now docker + + # Continue to Portainer section below + else + # Scenario 2: Use official Docker repository (USE_DOCKER_REPO=true) + + # Cleanup old repository configurations + if [ -f /etc/apt/sources.list.d/docker.list ]; then + msg_info "Migrating from old Docker repository format" + rm -f /etc/apt/sources.list.d/docker.list + rm -f /etc/apt/keyrings/docker.asc + fi + + # Setup/Update Docker repository + msg_info "Setting up Docker Repository" + setup_deb822_repo \ + "docker" \ + "https://download.docker.com/linux/$(get_os_info id)/gpg" \ + "https://download.docker.com/linux/$(get_os_info id)" \ + "$(get_os_info codename)" \ + "stable" \ + "$(dpkg --print-architecture)" + + # Install or upgrade Docker + if [ "$docker_installed" = true ]; then + msg_info "Checking for Docker updates" + DOCKER_LATEST_VERSION=$(apt-cache policy docker-ce | grep Candidate | awk '{print $2}' 2>/dev/null | cut -d':' -f2 | cut -d'-' -f1 || echo '') + + if [ "$DOCKER_CURRENT_VERSION" != "$DOCKER_LATEST_VERSION" ]; then + msg_info "Updating Docker $DOCKER_CURRENT_VERSION → $DOCKER_LATEST_VERSION" + $STD apt install -y --only-upgrade \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin || { + msg_error "Failed to update Docker packages" + return 100 + } + msg_ok "Updated Docker to $DOCKER_LATEST_VERSION" + else + msg_ok "Docker is up-to-date ($DOCKER_CURRENT_VERSION)" + fi + else + msg_info "Installing Docker" + $STD apt install -y \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin || { + msg_error "Failed to install Docker packages" + return 100 + } + + DOCKER_CURRENT_VERSION=$(docker --version | grep -oP '\d+\.\d+\.\d+' | head -1) + msg_ok "Installed Docker $DOCKER_CURRENT_VERSION" + fi + + # Configure daemon.json + local log_driver="${DOCKER_LOG_DRIVER:-journald}" + mkdir -p /etc/docker + if [ ! -f /etc/docker/daemon.json ]; then + cat </etc/docker/daemon.json +{ + "log-driver": "$log_driver" +} +EOF + fi + + # Enable and start Docker + systemctl enable -q --now docker + fi + + # Portainer Management (common for both modes) + if [[ "${DOCKER_PORTAINER:-}" == "true" ]]; then + if [ "$portainer_installed" = true ]; then + msg_info "Checking for Portainer updates" + PORTAINER_CURRENT=$(docker inspect portainer --format='{{.Config.Image}}' 2>/dev/null | cut -d':' -f2) + PORTAINER_LATEST=$(curl -fsSL https://registry.hub.docker.com/v2/repositories/portainer/portainer-ce/tags?page_size=100 | grep -oP '"name":"\K[0-9]+\.[0-9]+\.[0-9]+"' | head -1 | tr -d '"') + + if [ "$PORTAINER_CURRENT" != "$PORTAINER_LATEST" ]; then + read -r -p "${TAB3}Update Portainer $PORTAINER_CURRENT → $PORTAINER_LATEST? " prompt + if [[ ${prompt,,} =~ ^(y|yes)$ ]]; then + msg_info "Updating Portainer" + docker stop portainer + docker rm portainer + docker pull portainer/portainer-ce:latest + docker run -d \ + -p 9000:9000 \ + -p 9443:9443 \ + --name=portainer \ + --restart=always \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v portainer_data:/data \ + portainer/portainer-ce:latest + msg_ok "Updated Portainer to $PORTAINER_LATEST" + fi + else + msg_ok "Portainer is up-to-date ($PORTAINER_CURRENT)" + fi + else + msg_info "Installing Portainer" + docker volume create portainer_data + docker run -d \ + -p 9000:9000 \ + -p 9443:9443 \ + --name=portainer \ + --restart=always \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v portainer_data:/data \ + portainer/portainer-ce:latest + + LOCAL_IP=$(hostname -I | awk '{print $1}') + msg_ok "Installed Portainer (http://${LOCAL_IP}:9000)" + fi + fi + + # Interactive Container Update Check + if [[ "${DOCKER_SKIP_UPDATES:-}" != "true" ]] && [ "$docker_installed" = true ]; then + msg_info "Checking for container updates" + + # Get list of running containers with update status + local containers_with_updates=() + local container_info=() + local index=1 + + while IFS= read -r container; do + local name=$(echo "$container" | awk '{print $1}') + local image=$(echo "$container" | awk '{print $2}') + local current_digest=$(docker inspect "$name" --format='{{.Image}}' 2>/dev/null | cut -d':' -f2 | cut -c1-12) + + # Pull latest image digest + docker pull "$image" >/dev/null 2>&1 + local latest_digest=$(docker inspect "$image" --format='{{.Id}}' 2>/dev/null | cut -d':' -f2 | cut -c1-12) + + if [ "$current_digest" != "$latest_digest" ]; then + containers_with_updates+=("$name") + container_info+=("${index}) ${name} (${image})") + ((index++)) + fi + done < <(docker ps --format '{{.Names}} {{.Image}}') + + if [ ${#containers_with_updates[@]} -gt 0 ]; then + echo "" + echo "${TAB3}Container updates available:" + for info in "${container_info[@]}"; do + echo "${TAB3} $info" + done + echo "" + read -r -p "${TAB3}Select containers to update (e.g., 1,3,5 or 'all' or 'none'): " selection + + if [[ ${selection,,} == "all" ]]; then + for container in "${containers_with_updates[@]}"; do + msg_info "Updating container: $container" + docker stop "$container" + docker rm "$container" + # Note: This requires the original docker run command - best to recreate via compose + msg_ok "Stopped and removed $container (please recreate with updated image)" + done + elif [[ ${selection,,} != "none" ]]; then + IFS=',' read -ra SELECTED <<<"$selection" + for num in "${SELECTED[@]}"; do + num=$(echo "$num" | xargs) # trim whitespace + if [[ "$num" =~ ^[0-9]+$ ]] && [ "$num" -ge 1 ] && [ "$num" -le "${#containers_with_updates[@]}" ]; then + container="${containers_with_updates[$((num - 1))]}" + msg_info "Updating container: $container" + docker stop "$container" + docker rm "$container" + msg_ok "Stopped and removed $container (please recreate with updated image)" + fi + done + fi + else + msg_ok "All containers are up-to-date" + fi + fi + + msg_ok "Docker setup completed" +} + # ------------------------------------------------------------------------------ # Installs FFmpeg from source or prebuilt binary (Debian/Ubuntu only). # @@ -3936,7 +4441,7 @@ function setup_composer() { # - Result is installed to /usr/local/bin/ffmpeg # ------------------------------------------------------------------------------ -function setup_ffmpeg() { +setup_ffmpeg() { local TMP_DIR=$(mktemp -d) local GITHUB_REPO="FFmpeg/FFmpeg" local VERSION="${FFMPEG_VERSION:-latest}" @@ -3955,6 +4460,7 @@ function setup_ffmpeg() { if [[ "$TYPE" == "binary" ]]; then if ! CURL_TIMEOUT=300 curl_with_retry "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz" "$TMP_DIR/ffmpeg.tar.xz"; then msg_error "Failed to download FFmpeg binary" + msg_error "Hint: Check connectivity to johnvansickle.com/ffmpeg (static builds, may be slow — large file)" rm -rf "$TMP_DIR" return 250 fi @@ -4011,9 +4517,14 @@ function setup_ffmpeg() { DEPS+=( libx264-dev libx265-dev libvpx-dev libmp3lame-dev libfreetype6-dev libass-dev libopus-dev libvorbis-dev - libdav1d-dev libsvtav1-dev zlib1g-dev libnuma-dev + libdav1d-dev zlib1g-dev libnuma-dev libva-dev libdrm-dev ) + if apt-cache show libsvtav1enc-dev &>/dev/null; then + DEPS+=(libsvtav1enc-dev) + elif apt-cache show libsvtav1-dev &>/dev/null; then + DEPS+=(libsvtav1-dev) + fi ;; *) msg_error "Invalid FFMPEG_TYPE: $TYPE" @@ -4037,6 +4548,7 @@ function setup_ffmpeg() { msg_info "Setup FFmpeg from pre-built binary" if ! CURL_TIMEOUT=300 curl_with_retry "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz" "$TMP_DIR/ffmpeg.tar.xz"; then msg_error "Failed to download FFmpeg pre-built binary" + msg_error "Hint: Check connectivity to johnvansickle.com/ffmpeg (large file, may timeout on slow connections)" rm -rf "$TMP_DIR" return 250 fi @@ -4145,7 +4657,7 @@ function setup_ffmpeg() { # GO_VERSION - Version to install (e.g. 1.22.2 or latest) # ------------------------------------------------------------------------------ -function setup_go() { +setup_go() { local ARCH case "$(uname -m)" in x86_64) ARCH="amd64" ;; @@ -4197,6 +4709,7 @@ function setup_go() { if ! CURL_TIMEOUT=300 curl_with_retry "$URL" "$TMP_TAR"; then msg_error "Failed to download Go $GO_VERSION" + msg_error "Hint: Check connectivity to go.dev/dl — URL: $URL" rm -f "$TMP_TAR" return 250 fi @@ -4224,18 +4737,16 @@ function setup_go() { # - Builds and installs system-wide # ------------------------------------------------------------------------------ -function setup_gs() { +setup_gs() { local TMP_DIR=$(mktemp -d) local CURRENT_VERSION=$(gs --version 2>/dev/null || echo "0") ensure_dependencies jq - local RELEASE_JSON - RELEASE_JSON=$(curl -fsSL --max-time 15 https://api.github.com/repos/ArtifexSoftware/ghostpdl-downloads/releases/latest 2>/dev/null || echo "") - - if [[ -z "$RELEASE_JSON" ]]; then + local LATEST_VERSION + # Tags are "gs10.05.1" format — fetch without stripping "v", then strip "gs" + LATEST_VERSION=$(get_latest_github_release "ArtifexSoftware/ghostpdl-downloads" "false") || { msg_warn "Cannot fetch latest Ghostscript version from GitHub API" - # Try to get from current version if command -v gs &>/dev/null; then gs --version | head -n1 cache_installed_version "ghostscript" "$CURRENT_VERSION" @@ -4243,11 +4754,9 @@ function setup_gs() { fi msg_error "Cannot determine Ghostscript version and no existing installation found" return 250 - fi - local LATEST_VERSION - LATEST_VERSION=$(echo "$RELEASE_JSON" | jq -r '.tag_name' | sed 's/^gs//') - local LATEST_VERSION_DOTTED - LATEST_VERSION_DOTTED=$(echo "$RELEASE_JSON" | jq -r '.name' | grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+') + } + LATEST_VERSION="${LATEST_VERSION#gs}" + local LATEST_VERSION_DOTTED=$(echo "$LATEST_VERSION" | sed 's/\([0-9]\{2\}\)\([0-9]\{2\}\)\([0-9]\{1\}\)/\1.\2.\3/') if [[ -z "$LATEST_VERSION" || -z "$LATEST_VERSION_DOTTED" ]]; then msg_warn "Could not determine latest Ghostscript version from GitHub - checking system" @@ -4277,6 +4786,7 @@ function setup_gs() { if ! CURL_TIMEOUT=180 curl_with_retry "https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs${LATEST_VERSION}/ghostscript-${LATEST_VERSION_DOTTED}.tar.gz" "$TMP_DIR/ghostscript.tar.gz"; then msg_error "Failed to download Ghostscript" + msg_error "Hint: Check connectivity to github.com/ArtifexSoftware — may be a GitHub rate-limit, set GITHUB_TOKEN" rm -rf "$TMP_DIR" return 250 fi @@ -4347,7 +4857,7 @@ function setup_gs() { # - Some Intel packages are fetched from GitHub due to missing Debian packages # - NVIDIA requires matching host driver version # ------------------------------------------------------------------------------ -function setup_hwaccel() { +setup_hwaccel() { local service_user="${1:-}" # Check if user explicitly disabled GPU in advanced settings @@ -4533,9 +5043,9 @@ function setup_hwaccel() { # OS Detection # ═══════════════════════════════════════════════════════════════════════════ local os_id os_codename os_version - os_id=$(grep -oP '(?<=^ID=).+' /etc/os-release 2>/dev/null | tr -d '"' || echo "debian") - os_codename=$(grep -oP '(?<=^VERSION_CODENAME=).+' /etc/os-release 2>/dev/null | tr -d '"' || echo "unknown") - os_version=$(grep -oP '(?<=^VERSION_ID=).+' /etc/os-release 2>/dev/null | tr -d '"' || echo "") + os_id=$(get_os_info id) + os_codename=$(get_os_info codename) + os_version=$(get_os_info version) [[ -z "$os_id" ]] && os_id="debian" local in_ct="${CTTYPE:-0}" @@ -4607,6 +5117,23 @@ function setup_hwaccel() { msg_ok "Setup Hardware Acceleration" } +# ══════════════════════════════════════════════════════════════════════════════ +# Resolve the IGC tag that the latest compute-runtime was built against. +# Must be called AFTER a fetch_and_deploy_gh_release for intel/compute-runtime +# so that /tmp/gh_rel.json contains the compute-runtime release metadata. +# Sets the variable named by $1 (default: igc_tag) to the discovered tag. +# ══════════════════════════════════════════════════════════════════════════════ +_resolve_igc_tag() { + local -n _out_ref="${1:-igc_tag}" + _out_ref="latest" + if [[ -f /tmp/gh_rel.json ]]; then + local _body _parsed + _body=$(jq -r '.body // empty' /tmp/gh_rel.json 2>/dev/null) || return 0 + _parsed=$(grep -oP 'intel-graphics-compiler/releases/tag/\K[^\s\)]+' <<<"$_body" | head -1) + [[ -n "$_parsed" ]] && _out_ref="$_parsed" + fi +} + # ══════════════════════════════════════════════════════════════════════════════ # Intel Arc GPU Setup # ══════════════════════════════════════════════════════════════════════════════ @@ -4633,12 +5160,17 @@ _setup_intel_arc() { if [[ "$os_codename" == "trixie" || "$os_codename" == "sid" ]]; then msg_info "Fetching Intel compute-runtime from GitHub for Arc support" + # Fetch a compute-runtime package first so /tmp/gh_rel.json is populated, + # then resolve the matching IGC tag from the release notes. # libigdgmm - bundled in compute-runtime releases fetch_and_deploy_gh_release "libigdgmm12" "intel/compute-runtime" "binary" "latest" "" "libigdgmm12_*_amd64.deb" || true - # Intel Graphics Compiler (note: packages have -2 suffix) - fetch_and_deploy_gh_release "intel-igc-core" "intel/intel-graphics-compiler" "binary" "latest" "" "intel-igc-core-2_*_amd64.deb" || true - fetch_and_deploy_gh_release "intel-igc-opencl" "intel/intel-graphics-compiler" "binary" "latest" "" "intel-igc-opencl-2_*_amd64.deb" || true + local igc_tag + _resolve_igc_tag igc_tag + + # Intel Graphics Compiler – pinned to the version compute-runtime expects + fetch_and_deploy_gh_release "intel-igc-core" "intel/intel-graphics-compiler" "binary" "$igc_tag" "" "intel-igc-core-2_*_amd64.deb" || true + fetch_and_deploy_gh_release "intel-igc-opencl" "intel/intel-graphics-compiler" "binary" "$igc_tag" "" "intel-igc-opencl-2_*_amd64.deb" || true # Compute Runtime (depends on IGC and gmmlib) fetch_and_deploy_gh_release "intel-opencl-icd" "intel/compute-runtime" "binary" "latest" "" "intel-opencl-icd_*_amd64.deb" || true @@ -4688,12 +5220,17 @@ _setup_intel_modern() { if [[ "$os_codename" == "trixie" || "$os_codename" == "sid" ]]; then msg_info "Fetching Intel compute-runtime from GitHub" + # Fetch a compute-runtime package first so /tmp/gh_rel.json is populated, + # then resolve the matching IGC tag from the release notes. # libigdgmm first (bundled in compute-runtime releases) fetch_and_deploy_gh_release "libigdgmm12" "intel/compute-runtime" "binary" "latest" "" "libigdgmm12_*_amd64.deb" || true - # Intel Graphics Compiler (note: packages have -2 suffix) - fetch_and_deploy_gh_release "intel-igc-core" "intel/intel-graphics-compiler" "binary" "latest" "" "intel-igc-core-2_*_amd64.deb" || true - fetch_and_deploy_gh_release "intel-igc-opencl" "intel/intel-graphics-compiler" "binary" "latest" "" "intel-igc-opencl-2_*_amd64.deb" || true + local igc_tag + _resolve_igc_tag igc_tag + + # Intel Graphics Compiler – pinned to the version compute-runtime expects + fetch_and_deploy_gh_release "intel-igc-core" "intel/intel-graphics-compiler" "binary" "$igc_tag" "" "intel-igc-core-2_*_amd64.deb" || true + fetch_and_deploy_gh_release "intel-igc-opencl" "intel/intel-graphics-compiler" "binary" "$igc_tag" "" "intel-igc-opencl-2_*_amd64.deb" || true # Compute Runtime fetch_and_deploy_gh_release "intel-opencl-icd" "intel/compute-runtime" "binary" "latest" "" "intel-opencl-icd_*_amd64.deb" || true @@ -5322,7 +5859,7 @@ _setup_gpu_permissions() { # Notes: # - Requires: build-essential, libtool, libjpeg-dev, libpng-dev, etc. # ------------------------------------------------------------------------------ -function setup_imagemagick() { +setup_imagemagick() { local TMP_DIR=$(mktemp -d) local BINARY_PATH="/usr/local/bin/magick" @@ -5356,6 +5893,7 @@ function setup_imagemagick() { if ! CURL_TIMEOUT=180 curl_with_retry "https://imagemagick.org/archive/ImageMagick.tar.gz" "$TMP_DIR/ImageMagick.tar.gz"; then msg_error "Failed to download ImageMagick" + msg_error "Hint: Check connectivity to imagemagick.org/archive" rm -rf "$TMP_DIR" return 250 fi @@ -5419,11 +5957,11 @@ function setup_imagemagick() { # JAVA_VERSION - Temurin JDK version to install (e.g. 17, 21) # ------------------------------------------------------------------------------ -function setup_java() { +setup_java() { local JAVA_VERSION="${JAVA_VERSION:-21}" local DISTRO_ID DISTRO_CODENAME - DISTRO_ID=$(awk -F= '/^ID=/{print $2}' /etc/os-release | tr -d '"') - DISTRO_CODENAME=$(awk -F= '/VERSION_CODENAME/ { print $2 }' /etc/os-release) + DISTRO_ID=$(get_os_info id) + DISTRO_CODENAME=$(get_os_info codename) local DESIRED_PACKAGE="temurin-${JAVA_VERSION}-jdk" # Prepare repository (cleanup + validation) @@ -5489,7 +6027,7 @@ function setup_java() { # - Automatically runs on network changes # ------------------------------------------------------------------------------ -function setup_local_ip_helper() { +setup_local_ip_helper() { local BASE_DIR="/usr/local/community-scripts/ip-management" local SCRIPT_PATH="$BASE_DIR/update_local_ip.sh" local IP_FILE="/run/local-ip.env" @@ -5883,7 +6421,7 @@ _setup_mariadb_runtime_dir() { # MARIADB_DB_NAME, MARIADB_DB_USER, MARIADB_DB_PASS # ------------------------------------------------------------------------------ -function setup_mariadb_db() { +setup_mariadb_db() { if [[ -z "${MARIADB_DB_NAME:-}" || -z "${MARIADB_DB_USER:-}" ]]; then msg_error "MARIADB_DB_NAME and MARIADB_DB_USER must be set before calling setup_mariadb_db" return 65 @@ -5932,17 +6470,319 @@ function setup_mariadb_db() { } # ------------------------------------------------------------------------------ -# Installs or updates MongoDB to specified major version. +# Installs or updates MeiliSearch search engine. +# +# Description: +# - Fresh install: Downloads binary, creates config/service, starts +# - Update: Checks for new release, updates binary if available +# - Waits for service to be ready before returning +# - Exports API keys for use by caller +# +# Variables: +# MEILISEARCH_BIND - Bind address (default: 127.0.0.1:7700) +# MEILISEARCH_ENV - Environment: production/development (default: production) +# MEILISEARCH_DB_PATH - Database path (default: /var/lib/meilisearch/data) +# +# Exports: +# MEILISEARCH_MASTER_KEY - The master key for admin access +# MEILISEARCH_API_KEY - The default search API key +# MEILISEARCH_API_KEY_UID - The UID of the default API key +# +# Example (install script): +# setup_meilisearch +# +# Example (CT update_script): +# setup_meilisearch +# ------------------------------------------------------------------------------ + +setup_meilisearch() { + local MEILISEARCH_BIND="${MEILISEARCH_BIND:-127.0.0.1:7700}" + local MEILISEARCH_ENV="${MEILISEARCH_ENV:-production}" + local MEILISEARCH_DB_PATH="${MEILISEARCH_DB_PATH:-/var/lib/meilisearch/data}" + local MEILISEARCH_DUMP_DIR="${MEILISEARCH_DUMP_DIR:-/var/lib/meilisearch/dumps}" + local MEILISEARCH_SNAPSHOT_DIR="${MEILISEARCH_SNAPSHOT_DIR:-/var/lib/meilisearch/snapshots}" + + # Get bind address for health checks + local MEILISEARCH_HOST="${MEILISEARCH_BIND%%:*}" + local MEILISEARCH_PORT="${MEILISEARCH_BIND##*:}" + [[ "$MEILISEARCH_HOST" == "0.0.0.0" ]] && MEILISEARCH_HOST="127.0.0.1" + + # Update mode: MeiliSearch already installed + if [[ -f /usr/bin/meilisearch ]]; then + if check_for_gh_release "meilisearch" "meilisearch/meilisearch"; then + msg_info "Updating MeiliSearch" + + # Get current and new version for compatibility check + local CURRENT_VERSION NEW_VERSION + CURRENT_VERSION=$(/usr/bin/meilisearch --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) || CURRENT_VERSION="0.0.0" + NEW_VERSION="${CHECK_UPDATE_RELEASE#v}" + + # Extract major.minor for comparison (Meilisearch requires dump/restore between minor versions) + local CURRENT_MAJOR_MINOR NEW_MAJOR_MINOR + CURRENT_MAJOR_MINOR=$(echo "$CURRENT_VERSION" | cut -d. -f1,2) + NEW_MAJOR_MINOR=$(echo "$NEW_VERSION" | cut -d. -f1,2) + + # Determine if migration is needed (different major.minor = incompatible DB format) + local NEEDS_MIGRATION=false + if [[ "$CURRENT_MAJOR_MINOR" != "$NEW_MAJOR_MINOR" ]]; then + NEEDS_MIGRATION=true + msg_info "MeiliSearch version change detected (${CURRENT_VERSION} → ${NEW_VERSION}), preparing data migration" + fi + + # Read config values for dump/restore + local MEILI_HOST MEILI_PORT MEILI_MASTER_KEY MEILI_DUMP_DIR + MEILI_HOST="${MEILISEARCH_HOST:-127.0.0.1}" + MEILI_PORT="${MEILISEARCH_PORT:-7700}" + MEILI_DUMP_DIR="${MEILISEARCH_DUMP_DIR:-/var/lib/meilisearch/dumps}" + MEILI_MASTER_KEY=$(grep -E "^master_key\s*=" /etc/meilisearch.toml 2>/dev/null | sed 's/.*=\s*"\(.*\)"/\1/' | tr -d ' ' || true) + + # Create dump before update if migration is needed + local DUMP_UID="" + if [[ "$NEEDS_MIGRATION" == "true" ]] && [[ -n "$MEILI_MASTER_KEY" ]]; then + msg_info "Creating MeiliSearch data dump before upgrade" + + # Trigger dump creation + local DUMP_RESPONSE + DUMP_RESPONSE=$(curl -s -X POST "http://${MEILI_HOST}:${MEILI_PORT}/dumps" \ + -H "Authorization: Bearer ${MEILI_MASTER_KEY}" \ + -H "Content-Type: application/json" 2>/dev/null) || true + + # The initial response only contains taskUid, not dumpUid + # dumpUid is only available after the task completes + local TASK_UID + TASK_UID=$(echo "$DUMP_RESPONSE" | grep -oP '"taskUid":\s*\K[0-9]+' || true) + + if [[ -n "$TASK_UID" ]]; then + msg_info "Waiting for dump task ${TASK_UID} to complete..." + local MAX_WAIT=120 + local WAITED=0 + local TASK_RESULT="" + + while [[ $WAITED -lt $MAX_WAIT ]]; do + TASK_RESULT=$(curl -s "http://${MEILI_HOST}:${MEILI_PORT}/tasks/${TASK_UID}" \ + -H "Authorization: Bearer ${MEILI_MASTER_KEY}" 2>/dev/null) || true + + local TASK_STATUS + TASK_STATUS=$(echo "$TASK_RESULT" | grep -oP '"status":\s*"\K[^"]+' || true) + + if [[ "$TASK_STATUS" == "succeeded" ]]; then + # Extract dumpUid from the completed task details + DUMP_UID=$(echo "$TASK_RESULT" | grep -oP '"dumpUid":\s*"\K[^"]+' || true) + if [[ -n "$DUMP_UID" ]]; then + msg_ok "MeiliSearch dump created successfully: ${DUMP_UID}" + else + msg_warn "Dump task succeeded but could not extract dumpUid" + fi + break + elif [[ "$TASK_STATUS" == "failed" ]]; then + local ERROR_MSG + ERROR_MSG=$(echo "$TASK_RESULT" | grep -oP '"message":\s*"\K[^"]+' || echo "Unknown error") + msg_warn "MeiliSearch dump failed: ${ERROR_MSG}" + break + fi + sleep 2 + WAITED=$((WAITED + 2)) + done + + if [[ $WAITED -ge $MAX_WAIT ]]; then + msg_warn "MeiliSearch dump timed out after ${MAX_WAIT}s" + fi + else + msg_warn "Could not trigger MeiliSearch dump (no taskUid in response)" + msg_info "Response was: ${DUMP_RESPONSE:-empty}" + fi + fi + + # If migration is needed but dump failed, we have options: + # 1. Abort the update (safest, but annoying) + # 2. Backup data directory and proceed (allows manual recovery) + # 3. Just proceed and hope for the best (dangerous) + # We choose option 2: backup and proceed with warning + if [[ "$NEEDS_MIGRATION" == "true" ]] && [[ -z "$DUMP_UID" ]]; then + local MEILI_DB_PATH + MEILI_DB_PATH=$(grep -E "^db_path\s*=" /etc/meilisearch.toml 2>/dev/null | sed 's/.*=\s*"\(.*\)"/\1/' | tr -d ' ' || true) + MEILI_DB_PATH="${MEILI_DB_PATH:-/var/lib/meilisearch/data}" + + if [[ -d "$MEILI_DB_PATH" ]] && [[ -n "$(ls -A "$MEILI_DB_PATH" 2>/dev/null)" ]]; then + local BACKUP_PATH="${MEILI_DB_PATH}.backup.$(date +%Y%m%d%H%M%S)" + msg_warn "Backing up MeiliSearch data to ${BACKUP_PATH}" + mv "$MEILI_DB_PATH" "$BACKUP_PATH" + mkdir -p "$MEILI_DB_PATH" + msg_info "Data backed up. After update, you may need to reindex your data." + msg_info "Old data is preserved at: ${BACKUP_PATH}" + fi + fi + + # Stop service and update binary + systemctl stop meilisearch + fetch_and_deploy_gh_release "meilisearch" "meilisearch/meilisearch" "binary" + + # If migration needed and dump was created, remove old data and import dump + if [[ "$NEEDS_MIGRATION" == "true" ]] && [[ -n "$DUMP_UID" ]]; then + local MEILI_DB_PATH + MEILI_DB_PATH=$(grep -E "^db_path\s*=" /etc/meilisearch.toml 2>/dev/null | sed 's/.*=\s*"\(.*\)"/\1/' | tr -d ' ' || true) + MEILI_DB_PATH="${MEILI_DB_PATH:-/var/lib/meilisearch/data}" + + msg_info "Removing old MeiliSearch database for migration" + rm -rf "${MEILI_DB_PATH:?}"/* + + # Import dump using CLI flag (this is the supported method) + local DUMP_FILE="${MEILI_DUMP_DIR}/${DUMP_UID}.dump" + if [[ -f "$DUMP_FILE" ]]; then + msg_info "Importing dump: ${DUMP_FILE}" + + # Start meilisearch with --import-dump flag + # This is a one-time import that happens during startup + /usr/bin/meilisearch --config-file-path /etc/meilisearch.toml --import-dump "$DUMP_FILE" >/dev/null 2>&1 & + local MEILI_PID=$! + + # Wait for meilisearch to become healthy (import happens during startup) + msg_info "Waiting for MeiliSearch to import and start..." + local MAX_WAIT=300 + local WAITED=0 + while [[ $WAITED -lt $MAX_WAIT ]]; do + if curl -sf "http://${MEILI_HOST}:${MEILI_PORT}/health" &>/dev/null; then + msg_ok "MeiliSearch is healthy after import" + break + fi + # Check if process is still running + if ! kill -0 $MEILI_PID 2>/dev/null; then + msg_warn "MeiliSearch process exited during import" + break + fi + sleep 3 + WAITED=$((WAITED + 3)) + done + + # Stop the manual process + kill $MEILI_PID 2>/dev/null || true + wait $MEILI_PID 2>/dev/null || true + sleep 2 + + # Start via systemd for proper management + systemctl start meilisearch + + if systemctl is-active --quiet meilisearch; then + msg_ok "MeiliSearch migrated successfully" + else + msg_warn "MeiliSearch failed to start after migration - check logs with: journalctl -u meilisearch" + fi + else + msg_warn "Dump file not found: ${DUMP_FILE}" + systemctl start meilisearch + fi + else + systemctl start meilisearch + fi + + msg_ok "Updated MeiliSearch" + fi + return 0 + fi + + # Fresh install + msg_info "Setup MeiliSearch" + + # Install binary + fetch_and_deploy_gh_release "meilisearch" "meilisearch/meilisearch" "binary" || { + msg_error "Failed to install MeiliSearch binary" + return 250 + } + + # Download default config + curl -fsSL https://raw.githubusercontent.com/meilisearch/meilisearch/latest/config.toml -o /etc/meilisearch.toml || { + msg_error "Failed to download MeiliSearch config" + msg_error "Hint: Check connectivity to raw.githubusercontent.com/meilisearch/meilisearch" + return 7 + } + + # Generate master key + MEILISEARCH_MASTER_KEY=$(openssl rand -base64 12) + export MEILISEARCH_MASTER_KEY + + # Configure + sed -i \ + -e "s|^env =.*|env = \"${MEILISEARCH_ENV}\"|" \ + -e "s|^# master_key =.*|master_key = \"${MEILISEARCH_MASTER_KEY}\"|" \ + -e "s|^db_path =.*|db_path = \"${MEILISEARCH_DB_PATH}\"|" \ + -e "s|^dump_dir =.*|dump_dir = \"${MEILISEARCH_DUMP_DIR}\"|" \ + -e "s|^snapshot_dir =.*|snapshot_dir = \"${MEILISEARCH_SNAPSHOT_DIR}\"|" \ + -e 's|^# no_analytics = true|no_analytics = true|' \ + -e "s|^http_addr =.*|http_addr = \"${MEILISEARCH_BIND}\"|" \ + /etc/meilisearch.toml + + # Create data directories + mkdir -p "${MEILISEARCH_DB_PATH}" "${MEILISEARCH_DUMP_DIR}" "${MEILISEARCH_SNAPSHOT_DIR}" + + # Create systemd service + cat </etc/systemd/system/meilisearch.service +[Unit] +Description=Meilisearch +After=network.target + +[Service] +ExecStart=/usr/bin/meilisearch --config-file-path /etc/meilisearch.toml +Restart=always + +[Install] +WantedBy=multi-user.target +EOF + + # Enable and start service + systemctl daemon-reload + systemctl enable -q --now meilisearch + + # Wait for MeiliSearch to be ready (up to 30 seconds) + for i in {1..30}; do + if curl -s -o /dev/null -w "%{http_code}" "http://${MEILISEARCH_HOST}:${MEILISEARCH_PORT}/health" 2>/dev/null | grep -q "200"; then + break + fi + sleep 1 + done + + # Verify service is running + if ! systemctl is-active --quiet meilisearch; then + msg_error "MeiliSearch service failed to start" + return 150 + fi + + # Get API keys with retry logic + MEILISEARCH_API_KEY="" + for i in {1..10}; do + MEILISEARCH_API_KEY=$(curl -s -X GET "http://${MEILISEARCH_HOST}:${MEILISEARCH_PORT}/keys" \ + -H "Authorization: Bearer ${MEILISEARCH_MASTER_KEY}" 2>/dev/null | + grep -o '"key":"[^"]*"' | head -n 1 | sed 's/"key":"//;s/"//') || true + [[ -n "$MEILISEARCH_API_KEY" ]] && break + sleep 2 + done + + MEILISEARCH_API_KEY_UID=$(curl -s -X GET "http://${MEILISEARCH_HOST}:${MEILISEARCH_PORT}/keys" \ + -H "Authorization: Bearer ${MEILISEARCH_MASTER_KEY}" 2>/dev/null | + grep -o '"uid":"[^"]*"' | head -n 1 | sed 's/"uid":"//;s/"//') || true + + export MEILISEARCH_API_KEY + export MEILISEARCH_API_KEY_UID + + # Cache version + local MEILISEARCH_VERSION + MEILISEARCH_VERSION=$(/usr/bin/meilisearch --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) || true + cache_installed_version "meilisearch" "${MEILISEARCH_VERSION:-unknown}" + + msg_ok "Setup MeiliSearch ${MEILISEARCH_VERSION:-}" +} + +# ------------------------------------------------------------------------------ +# Installs or updates MongoDB to specified version. # # Description: # - Preserves data across installations # - Adds official MongoDB repo # # Variables: -# MONGO_VERSION - MongoDB major version to install (e.g. 7.0, 8.0) +# MONGO_VERSION - MongoDB version to install (e.g. 7.0, 8.2) # ------------------------------------------------------------------------------ -function setup_mongodb() { +setup_mongodb() { local MONGO_VERSION="${MONGO_VERSION:-8.0}" local DISTRO_ID DISTRO_CODENAME DISTRO_ID=$(get_os_info id) @@ -6012,8 +6852,11 @@ function setup_mongodb() { } # Setup repository + # MongoDB 8.x versions beyond 8.0 reuse the server-8.0.asc PGP key + local MONGO_KEY_VERSION="${MONGO_VERSION}" + [[ "${MONGO_VERSION}" == 8.[1-9]* ]] && MONGO_KEY_VERSION="8.0" manage_tool_repository "mongodb" "$MONGO_VERSION" "$MONGO_BASE_URL" \ - "https://www.mongodb.org/static/pgp/server-${MONGO_VERSION}.asc" || { + "https://www.mongodb.org/static/pgp/server-${MONGO_KEY_VERSION}.asc" || { msg_error "Failed to setup MongoDB repository" return 100 } @@ -6075,12 +6918,12 @@ function setup_mongodb() { # USE_MYSQL_REPO=false setup_mysql # Uses distro package instead # ------------------------------------------------------------------------------ -function setup_mysql() { +setup_mysql() { local MYSQL_VERSION="${MYSQL_VERSION:-8.0}" local USE_MYSQL_REPO="${USE_MYSQL_REPO:-true}" local DISTRO_ID DISTRO_CODENAME - DISTRO_ID=$(awk -F= '/^ID=/{print $2}' /etc/os-release | tr -d '"') - DISTRO_CODENAME=$(awk -F= '/^VERSION_CODENAME=/{print $2}' /etc/os-release) + DISTRO_ID=$(get_os_info id) + DISTRO_CODENAME=$(get_os_info codename) # Ensure non-interactive mode for all apt operations export DEBIAN_FRONTEND=noninteractive @@ -6289,7 +7132,7 @@ EOF # NODE_MODULE - Comma-separated list of global modules (e.g. "yarn,@vue/cli@5.0.0") # ------------------------------------------------------------------------------ -function setup_nodejs() { +setup_nodejs() { local NODE_VERSION="${NODE_VERSION:-24}" local NODE_MODULE="${NODE_MODULE:-}" @@ -6309,12 +7152,15 @@ function setup_nodejs() { } fi - # Scenario 1: Already installed at target version - just update packages/modules + # Scenario 1: Already installed at target version - upgrade to latest minor/patch + update packages/modules if [[ -n "$CURRENT_NODE_VERSION" && "$CURRENT_NODE_VERSION" == "$NODE_VERSION" ]]; then msg_info "Update Node.js $NODE_VERSION" ensure_apt_working || return 100 + # Upgrade to the latest minor/patch release from NodeSource + $STD apt-get install -y --only-upgrade nodejs 2>/dev/null || true + # Pin npm to 11.11.0 to work around Node.js 22.22.2 regression (nodejs/node#62425) $STD npm install -g npm@11.11.0 2>/dev/null || true @@ -6393,7 +7239,37 @@ function setup_nodejs() { msg_ok "Setup Node.js $NODE_VERSION" fi - export NODE_OPTIONS="--max-old-space-size=4096" + # Set a safe default heap limit for Node.js builds if not explicitly provided. + # Priority: + # 1) NODE_OPTIONS (caller/user override) + # 2) NODE_MAX_OLD_SPACE_SIZE (explicit MB override) + # 3) var_ram (LXC memory setting, MB) + # 4) /proc/meminfo (runtime memory detection) + # Auto value is clamped to 1024..12288 MB. + if [[ -z "${NODE_OPTIONS:-}" ]]; then + local node_heap_mb="" + + if [[ -n "${NODE_MAX_OLD_SPACE_SIZE:-}" ]] && [[ "${NODE_MAX_OLD_SPACE_SIZE}" =~ ^[0-9]+$ ]]; then + node_heap_mb="${NODE_MAX_OLD_SPACE_SIZE}" + elif [[ -n "${var_ram:-}" ]] && [[ "${var_ram}" =~ ^[0-9]+$ ]]; then + node_heap_mb=$((var_ram * 75 / 100)) + else + local total_mem_kb="" + total_mem_kb=$(awk '/^MemTotal:/ {print $2; exit}' /proc/meminfo 2>/dev/null || echo "") + if [[ "$total_mem_kb" =~ ^[0-9]+$ ]]; then + local total_mem_mb=$((total_mem_kb / 1024)) + node_heap_mb=$((total_mem_mb * 75 / 100)) + fi + fi + + if [[ -z "$node_heap_mb" ]] || ((node_heap_mb < 1024)); then + node_heap_mb=1024 + elif ((node_heap_mb > 12288)); then + node_heap_mb=12288 + fi + + export NODE_OPTIONS="--max-old-space-size=${node_heap_mb}" + fi # Ensure valid working directory for npm (avoids uv_cwd error) if [[ ! -d /opt ]]; then @@ -6407,6 +7283,14 @@ function setup_nodejs() { # Install global Node modules if [[ -n "$NODE_MODULE" ]]; then IFS=',' read -ra MODULES <<<"$NODE_MODULE" + + # Pin pnpm to v10 to avoid breaking changes from newer major versions + for i in "${!MODULES[@]}"; do + if [[ "${MODULES[$i]}" =~ ^pnpm(@.*)?$ ]]; then + MODULES[$i]="pnpm@^10" + fi + done + local failed_modules=0 for mod in "${MODULES[@]}"; do local MODULE_NAME MODULE_REQ_VERSION MODULE_INSTALLED_VERSION @@ -6491,14 +7375,14 @@ function setup_nodejs() { # - Unavailable modules are skipped with a warning, not an error # ------------------------------------------------------------------------------ -function setup_php() { +setup_php() { local PHP_VERSION="${PHP_VERSION:-8.4}" local PHP_MODULE="${PHP_MODULE:-}" local PHP_APACHE="${PHP_APACHE:-NO}" local PHP_FPM="${PHP_FPM:-NO}" local DISTRO_ID DISTRO_CODENAME - DISTRO_ID=$(awk -F= '/^ID=/{print $2}' /etc/os-release | tr -d '"') - DISTRO_CODENAME=$(awk -F= '/^VERSION_CODENAME=/{print $2}' /etc/os-release) + DISTRO_ID=$(get_os_info id) + DISTRO_CODENAME=$(get_os_info codename) # Parse version for compatibility checks local PHP_MAJOR="${PHP_VERSION%%.*}" @@ -6810,8 +7694,8 @@ setup_postgresql() { local PG_MODULES="${PG_MODULES:-}" local USE_PGDG_REPO="${USE_PGDG_REPO:-true}" local DISTRO_ID DISTRO_CODENAME - DISTRO_ID=$(awk -F= '/^ID=/{print $2}' /etc/os-release | tr -d '"') - DISTRO_CODENAME=$(awk -F= '/^VERSION_CODENAME=/{print $2}' /etc/os-release) + DISTRO_ID=$(get_os_info id) + DISTRO_CODENAME=$(get_os_info codename) # Ensure non-interactive mode for all apt operations export DEBIAN_FRONTEND=noninteractive @@ -6840,7 +7724,8 @@ setup_postgresql() { if [[ -n "$PG_MODULES" ]]; then IFS=',' read -ra MODULES <<<"$PG_MODULES" for module in "${MODULES[@]}"; do - $STD apt install -y "postgresql-${CURRENT_PG_VERSION}-${module}" 2>/dev/null || true + $STD apt install -y "postgresql-${CURRENT_PG_VERSION}-${module}" 2>/dev/null || + msg_warn "Optional PostgreSQL module '${module}' not available for PG ${CURRENT_PG_VERSION} on $(get_os_info codename) — skipping" done fi _configure_pg_cron_preload "$PG_MODULES" @@ -6876,7 +7761,8 @@ setup_postgresql() { if [[ -n "$PG_MODULES" && -n "$INSTALLED_VERSION" ]]; then IFS=',' read -ra MODULES <<<"$PG_MODULES" for module in "${MODULES[@]}"; do - $STD apt install -y "postgresql-${INSTALLED_VERSION}-${module}" 2>/dev/null || true + $STD apt install -y "postgresql-${INSTALLED_VERSION}-${module}" 2>/dev/null || + msg_warn "Optional PostgreSQL module '${module}' not available for PG ${INSTALLED_VERSION} on $(get_os_info codename) — skipping" done fi _configure_pg_cron_preload "$PG_MODULES" @@ -6898,7 +7784,8 @@ setup_postgresql() { if [[ -n "$PG_MODULES" ]]; then IFS=',' read -ra MODULES <<<"$PG_MODULES" for module in "${MODULES[@]}"; do - $STD apt install -y "postgresql-${PG_VERSION}-${module}" 2>/dev/null || true + $STD apt install -y "postgresql-${PG_VERSION}-${module}" 2>/dev/null || + msg_warn "Optional PostgreSQL module '${module}' not available for PG ${PG_VERSION} on $(get_os_info codename) — skipping" done fi _configure_pg_cron_preload "$PG_MODULES" @@ -6934,7 +7821,13 @@ setup_postgresql() { SUITE="trixie-pgdg" else - msg_warn "PGDG repo not available for ${DISTRO_CODENAME}, falling back to distro packages" + local _distro_pg_ver + _distro_pg_ver=$(apt-cache show postgresql 2>/dev/null | awk '/^Version:/{print $2; exit}' | grep -oE '^[0-9]+' || true) + msg_warn "PGDG repository not available for ${DISTRO_CODENAME} — falling back to distro-provided PostgreSQL packages" + if [[ -n "$_distro_pg_ver" ]]; then + msg_warn "Distro will install PostgreSQL ${_distro_pg_ver} (not the requested ${PG_VERSION})." + msg_warn "Any PostgreSQL extension packages (e.g. vchord, pgvector) must be built for PostgreSQL ${_distro_pg_ver} on ${DISTRO_CODENAME}." + fi USE_PGDG_REPO=false setup_postgresql return $? fi @@ -6960,7 +7853,8 @@ setup_postgresql() { # Install ssl-cert dependency if available if apt-cache search "^ssl-cert$" 2>/dev/null | grep -q .; then - $STD apt install -y ssl-cert 2>/dev/null || true + $STD apt install -y ssl-cert 2>/dev/null || + msg_warn "Optional ssl-cert package could not be installed — continuing without it" fi # Try multiple PostgreSQL package patterns with retry logic @@ -7057,7 +7951,7 @@ setup_postgresql() { # PG_DB_NAME, PG_DB_USER, PG_DB_PASS - For use in calling script # ------------------------------------------------------------------------------ -function setup_postgresql_db() { +setup_postgresql_db() { # Validation if [[ -z "${PG_DB_NAME:-}" || -z "${PG_DB_USER:-}" ]]; then msg_error "PG_DB_NAME and PG_DB_USER must be set before calling setup_postgresql_db" @@ -7149,7 +8043,7 @@ function setup_postgresql_db() { # RUBY_INSTALL_RAILS - true/false to install Rails (default: true) # ------------------------------------------------------------------------------ -function setup_ruby() { +setup_ruby() { local RUBY_VERSION="${RUBY_VERSION:-3.4.4}" local RUBY_INSTALL_RAILS="${RUBY_INSTALL_RAILS:-true}" local RBENV_DIR="$HOME/.rbenv" @@ -7216,7 +8110,8 @@ function setup_ruby() { done if [[ ${#ruby_deps[@]} -gt 0 ]]; then - $STD apt install -y "${ruby_deps[@]}" 2>/dev/null || true + $STD apt install -y "${ruby_deps[@]}" 2>/dev/null || + msg_warn "Some Ruby build dependencies could not be installed — compilation may fail" else msg_error "No Ruby build dependencies available" rm -rf "$TMP_DIR" @@ -7226,25 +8121,15 @@ function setup_ruby() { # Download and build rbenv if needed if [[ ! -x "$RBENV_BIN" ]]; then local RBENV_RELEASE - local rbenv_json - rbenv_json=$(curl -fsSL --max-time 15 https://api.github.com/repos/rbenv/rbenv/releases/latest 2>/dev/null || echo "") - - if [[ -z "$rbenv_json" ]]; then + RBENV_RELEASE=$(get_latest_github_release "rbenv/rbenv") || { msg_error "Failed to fetch latest rbenv version from GitHub" rm -rf "$TMP_DIR" return 7 - fi - - RBENV_RELEASE=$(echo "$rbenv_json" | jq -r '.tag_name' 2>/dev/null | sed 's/^v//' || echo "") - - if [[ -z "$RBENV_RELEASE" ]]; then - msg_error "Could not parse rbenv version from GitHub response" - rm -rf "$TMP_DIR" - return 250 - fi + } if ! curl_with_retry "https://github.com/rbenv/rbenv/archive/refs/tags/v${RBENV_RELEASE}.tar.gz" "$TMP_DIR/rbenv.tar.gz"; then msg_error "Failed to download rbenv" + msg_error "Hint: Check connectivity to github.com/rbenv/rbenv" rm -rf "$TMP_DIR" return 7 fi @@ -7273,25 +8158,15 @@ function setup_ruby() { # Install ruby-build plugin if [[ ! -d "$RBENV_DIR/plugins/ruby-build" ]]; then local RUBY_BUILD_RELEASE - local ruby_build_json - ruby_build_json=$(curl -fsSL --max-time 15 https://api.github.com/repos/rbenv/ruby-build/releases/latest 2>/dev/null || echo "") - - if [[ -z "$ruby_build_json" ]]; then + RUBY_BUILD_RELEASE=$(get_latest_github_release "rbenv/ruby-build") || { msg_error "Failed to fetch latest ruby-build version from GitHub" rm -rf "$TMP_DIR" return 7 - fi - - RUBY_BUILD_RELEASE=$(echo "$ruby_build_json" | jq -r '.tag_name' 2>/dev/null | sed 's/^v//' || echo "") - - if [[ -z "$RUBY_BUILD_RELEASE" ]]; then - msg_error "Could not parse ruby-build version from GitHub response" - rm -rf "$TMP_DIR" - return 250 - fi + } if ! curl_with_retry "https://github.com/rbenv/ruby-build/archive/refs/tags/v${RUBY_BUILD_RELEASE}.tar.gz" "$TMP_DIR/ruby-build.tar.gz"; then msg_error "Failed to download ruby-build" + msg_error "Hint: Check connectivity to github.com/rbenv/ruby-build" rm -rf "$TMP_DIR" return 7 fi @@ -7338,426 +8213,6 @@ function setup_ruby() { msg_ok "Setup Ruby $RUBY_VERSION" } -# ------------------------------------------------------------------------------ -# Installs or updates MeiliSearch search engine. -# -# Description: -# - Fresh install: Downloads binary, creates config/service, starts -# - Update: Checks for new release, updates binary if available -# - Waits for service to be ready before returning -# - Exports API keys for use by caller -# -# Variables: -# MEILISEARCH_BIND - Bind address (default: 127.0.0.1:7700) -# MEILISEARCH_ENV - Environment: production/development (default: production) -# MEILISEARCH_DB_PATH - Database path (default: /var/lib/meilisearch/data) -# -# Exports: -# MEILISEARCH_MASTER_KEY - The master key for admin access -# MEILISEARCH_API_KEY - The default search API key -# MEILISEARCH_API_KEY_UID - The UID of the default API key -# -# Example (install script): -# setup_meilisearch -# -# Example (CT update_script): -# setup_meilisearch -# ------------------------------------------------------------------------------ - -function setup_meilisearch() { - local MEILISEARCH_BIND="${MEILISEARCH_BIND:-127.0.0.1:7700}" - local MEILISEARCH_ENV="${MEILISEARCH_ENV:-production}" - local MEILISEARCH_DB_PATH="${MEILISEARCH_DB_PATH:-/var/lib/meilisearch/data}" - local MEILISEARCH_DUMP_DIR="${MEILISEARCH_DUMP_DIR:-/var/lib/meilisearch/dumps}" - local MEILISEARCH_SNAPSHOT_DIR="${MEILISEARCH_SNAPSHOT_DIR:-/var/lib/meilisearch/snapshots}" - - # Get bind address for health checks - local MEILISEARCH_HOST="${MEILISEARCH_BIND%%:*}" - local MEILISEARCH_PORT="${MEILISEARCH_BIND##*:}" - [[ "$MEILISEARCH_HOST" == "0.0.0.0" ]] && MEILISEARCH_HOST="127.0.0.1" - - # Update mode: MeiliSearch already installed - if [[ -f /usr/bin/meilisearch ]]; then - if check_for_gh_release "meilisearch" "meilisearch/meilisearch"; then - msg_info "Updating MeiliSearch" - - # Get current and new version for compatibility check - local CURRENT_VERSION NEW_VERSION - CURRENT_VERSION=$(/usr/bin/meilisearch --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) || CURRENT_VERSION="0.0.0" - NEW_VERSION="${CHECK_UPDATE_RELEASE#v}" - - # Extract major.minor for comparison (Meilisearch requires dump/restore between minor versions) - local CURRENT_MAJOR_MINOR NEW_MAJOR_MINOR - CURRENT_MAJOR_MINOR=$(echo "$CURRENT_VERSION" | cut -d. -f1,2) - NEW_MAJOR_MINOR=$(echo "$NEW_VERSION" | cut -d. -f1,2) - - # Determine if migration is needed (different major.minor = incompatible DB format) - local NEEDS_MIGRATION=false - if [[ "$CURRENT_MAJOR_MINOR" != "$NEW_MAJOR_MINOR" ]]; then - NEEDS_MIGRATION=true - msg_info "MeiliSearch version change detected (${CURRENT_VERSION} → ${NEW_VERSION}), preparing data migration" - fi - - # Read config values for dump/restore - local MEILI_HOST MEILI_PORT MEILI_MASTER_KEY MEILI_DUMP_DIR - MEILI_HOST="${MEILISEARCH_HOST:-127.0.0.1}" - MEILI_PORT="${MEILISEARCH_PORT:-7700}" - MEILI_DUMP_DIR="${MEILISEARCH_DUMP_DIR:-/var/lib/meilisearch/dumps}" - MEILI_MASTER_KEY=$(grep -E "^master_key\s*=" /etc/meilisearch.toml 2>/dev/null | sed 's/.*=\s*"\(.*\)"/\1/' | tr -d ' ' || true) - - # Create dump before update if migration is needed - local DUMP_UID="" - if [[ "$NEEDS_MIGRATION" == "true" ]] && [[ -n "$MEILI_MASTER_KEY" ]]; then - msg_info "Creating MeiliSearch data dump before upgrade" - - # Trigger dump creation - local DUMP_RESPONSE - DUMP_RESPONSE=$(curl -s -X POST "http://${MEILI_HOST}:${MEILI_PORT}/dumps" \ - -H "Authorization: Bearer ${MEILI_MASTER_KEY}" \ - -H "Content-Type: application/json" 2>/dev/null) || true - - # The initial response only contains taskUid, not dumpUid - # dumpUid is only available after the task completes - local TASK_UID - TASK_UID=$(echo "$DUMP_RESPONSE" | grep -oP '"taskUid":\s*\K[0-9]+' || true) - - if [[ -n "$TASK_UID" ]]; then - msg_info "Waiting for dump task ${TASK_UID} to complete..." - local MAX_WAIT=120 - local WAITED=0 - local TASK_RESULT="" - - while [[ $WAITED -lt $MAX_WAIT ]]; do - TASK_RESULT=$(curl -s "http://${MEILI_HOST}:${MEILI_PORT}/tasks/${TASK_UID}" \ - -H "Authorization: Bearer ${MEILI_MASTER_KEY}" 2>/dev/null) || true - - local TASK_STATUS - TASK_STATUS=$(echo "$TASK_RESULT" | grep -oP '"status":\s*"\K[^"]+' || true) - - if [[ "$TASK_STATUS" == "succeeded" ]]; then - # Extract dumpUid from the completed task details - DUMP_UID=$(echo "$TASK_RESULT" | grep -oP '"dumpUid":\s*"\K[^"]+' || true) - if [[ -n "$DUMP_UID" ]]; then - msg_ok "MeiliSearch dump created successfully: ${DUMP_UID}" - else - msg_warn "Dump task succeeded but could not extract dumpUid" - fi - break - elif [[ "$TASK_STATUS" == "failed" ]]; then - local ERROR_MSG - ERROR_MSG=$(echo "$TASK_RESULT" | grep -oP '"message":\s*"\K[^"]+' || echo "Unknown error") - msg_warn "MeiliSearch dump failed: ${ERROR_MSG}" - break - fi - sleep 2 - WAITED=$((WAITED + 2)) - done - - if [[ $WAITED -ge $MAX_WAIT ]]; then - msg_warn "MeiliSearch dump timed out after ${MAX_WAIT}s" - fi - else - msg_warn "Could not trigger MeiliSearch dump (no taskUid in response)" - msg_info "Response was: ${DUMP_RESPONSE:-empty}" - fi - fi - - # If migration is needed but dump failed, we have options: - # 1. Abort the update (safest, but annoying) - # 2. Backup data directory and proceed (allows manual recovery) - # 3. Just proceed and hope for the best (dangerous) - # We choose option 2: backup and proceed with warning - if [[ "$NEEDS_MIGRATION" == "true" ]] && [[ -z "$DUMP_UID" ]]; then - local MEILI_DB_PATH - MEILI_DB_PATH=$(grep -E "^db_path\s*=" /etc/meilisearch.toml 2>/dev/null | sed 's/.*=\s*"\(.*\)"/\1/' | tr -d ' ' || true) - MEILI_DB_PATH="${MEILI_DB_PATH:-/var/lib/meilisearch/data}" - - if [[ -d "$MEILI_DB_PATH" ]] && [[ -n "$(ls -A "$MEILI_DB_PATH" 2>/dev/null)" ]]; then - local BACKUP_PATH="${MEILI_DB_PATH}.backup.$(date +%Y%m%d%H%M%S)" - msg_warn "Backing up MeiliSearch data to ${BACKUP_PATH}" - mv "$MEILI_DB_PATH" "$BACKUP_PATH" - mkdir -p "$MEILI_DB_PATH" - msg_info "Data backed up. After update, you may need to reindex your data." - msg_info "Old data is preserved at: ${BACKUP_PATH}" - fi - fi - - # Stop service and update binary - systemctl stop meilisearch - fetch_and_deploy_gh_release "meilisearch" "meilisearch/meilisearch" "binary" - - # If migration needed and dump was created, remove old data and import dump - if [[ "$NEEDS_MIGRATION" == "true" ]] && [[ -n "$DUMP_UID" ]]; then - local MEILI_DB_PATH - MEILI_DB_PATH=$(grep -E "^db_path\s*=" /etc/meilisearch.toml 2>/dev/null | sed 's/.*=\s*"\(.*\)"/\1/' | tr -d ' ' || true) - MEILI_DB_PATH="${MEILI_DB_PATH:-/var/lib/meilisearch/data}" - - msg_info "Removing old MeiliSearch database for migration" - rm -rf "${MEILI_DB_PATH:?}"/* - - # Import dump using CLI flag (this is the supported method) - local DUMP_FILE="${MEILI_DUMP_DIR}/${DUMP_UID}.dump" - if [[ -f "$DUMP_FILE" ]]; then - msg_info "Importing dump: ${DUMP_FILE}" - - # Start meilisearch with --import-dump flag - # This is a one-time import that happens during startup - /usr/bin/meilisearch --config-file-path /etc/meilisearch.toml --import-dump "$DUMP_FILE" & - local MEILI_PID=$! - - # Wait for meilisearch to become healthy (import happens during startup) - msg_info "Waiting for MeiliSearch to import and start..." - local MAX_WAIT=300 - local WAITED=0 - while [[ $WAITED -lt $MAX_WAIT ]]; do - if curl -sf "http://${MEILI_HOST}:${MEILI_PORT}/health" &>/dev/null; then - msg_ok "MeiliSearch is healthy after import" - break - fi - # Check if process is still running - if ! kill -0 $MEILI_PID 2>/dev/null; then - msg_warn "MeiliSearch process exited during import" - break - fi - sleep 3 - WAITED=$((WAITED + 3)) - done - - # Stop the manual process - kill $MEILI_PID 2>/dev/null || true - sleep 2 - - # Start via systemd for proper management - systemctl start meilisearch - - if systemctl is-active --quiet meilisearch; then - msg_ok "MeiliSearch migrated successfully" - else - msg_warn "MeiliSearch failed to start after migration - check logs with: journalctl -u meilisearch" - fi - else - msg_warn "Dump file not found: ${DUMP_FILE}" - systemctl start meilisearch - fi - else - systemctl start meilisearch - fi - - msg_ok "Updated MeiliSearch" - fi - return 0 - fi - - # Fresh install - msg_info "Setup MeiliSearch" - - # Install binary - fetch_and_deploy_gh_release "meilisearch" "meilisearch/meilisearch" "binary" || { - msg_error "Failed to install MeiliSearch binary" - return 250 - } - - # Download default config - curl -fsSL https://raw.githubusercontent.com/meilisearch/meilisearch/latest/config.toml -o /etc/meilisearch.toml || { - msg_error "Failed to download MeiliSearch config" - return 7 - } - - # Generate master key - MEILISEARCH_MASTER_KEY=$(openssl rand -base64 12) - export MEILISEARCH_MASTER_KEY - - # Configure - sed -i \ - -e "s|^env =.*|env = \"${MEILISEARCH_ENV}\"|" \ - -e "s|^# master_key =.*|master_key = \"${MEILISEARCH_MASTER_KEY}\"|" \ - -e "s|^db_path =.*|db_path = \"${MEILISEARCH_DB_PATH}\"|" \ - -e "s|^dump_dir =.*|dump_dir = \"${MEILISEARCH_DUMP_DIR}\"|" \ - -e "s|^snapshot_dir =.*|snapshot_dir = \"${MEILISEARCH_SNAPSHOT_DIR}\"|" \ - -e 's|^# no_analytics = true|no_analytics = true|' \ - -e "s|^http_addr =.*|http_addr = \"${MEILISEARCH_BIND}\"|" \ - /etc/meilisearch.toml - - # Create data directories - mkdir -p "${MEILISEARCH_DB_PATH}" "${MEILISEARCH_DUMP_DIR}" "${MEILISEARCH_SNAPSHOT_DIR}" - - # Create systemd service - cat </etc/systemd/system/meilisearch.service -[Unit] -Description=Meilisearch -After=network.target - -[Service] -ExecStart=/usr/bin/meilisearch --config-file-path /etc/meilisearch.toml -Restart=always - -[Install] -WantedBy=multi-user.target -EOF - - # Enable and start service - systemctl daemon-reload - systemctl enable -q --now meilisearch - - # Wait for MeiliSearch to be ready (up to 30 seconds) - for i in {1..30}; do - if curl -s -o /dev/null -w "%{http_code}" "http://${MEILISEARCH_HOST}:${MEILISEARCH_PORT}/health" 2>/dev/null | grep -q "200"; then - break - fi - sleep 1 - done - - # Verify service is running - if ! systemctl is-active --quiet meilisearch; then - msg_error "MeiliSearch service failed to start" - return 150 - fi - - # Get API keys with retry logic - MEILISEARCH_API_KEY="" - for i in {1..10}; do - MEILISEARCH_API_KEY=$(curl -s -X GET "http://${MEILISEARCH_HOST}:${MEILISEARCH_PORT}/keys" \ - -H "Authorization: Bearer ${MEILISEARCH_MASTER_KEY}" 2>/dev/null | - grep -o '"key":"[^"]*"' | head -n 1 | sed 's/"key":"//;s/"//') || true - [[ -n "$MEILISEARCH_API_KEY" ]] && break - sleep 2 - done - - MEILISEARCH_API_KEY_UID=$(curl -s -X GET "http://${MEILISEARCH_HOST}:${MEILISEARCH_PORT}/keys" \ - -H "Authorization: Bearer ${MEILISEARCH_MASTER_KEY}" 2>/dev/null | - grep -o '"uid":"[^"]*"' | head -n 1 | sed 's/"uid":"//;s/"//') || true - - export MEILISEARCH_API_KEY - export MEILISEARCH_API_KEY_UID - - # Cache version - local MEILISEARCH_VERSION - MEILISEARCH_VERSION=$(/usr/bin/meilisearch --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) || true - cache_installed_version "meilisearch" "${MEILISEARCH_VERSION:-unknown}" - - msg_ok "Setup MeiliSearch ${MEILISEARCH_VERSION:-}" -} - -# ------------------------------------------------------------------------------ -# Installs or upgrades ClickHouse database server. -# -# Description: -# - Adds ClickHouse official repository -# - Installs specified version -# - Configures systemd service -# - Supports Debian/Ubuntu with fallback mechanism -# -# Variables: -# CLICKHOUSE_VERSION - ClickHouse version to install (default: latest) -# ------------------------------------------------------------------------------ - -function setup_clickhouse() { - local CLICKHOUSE_VERSION="${CLICKHOUSE_VERSION:-latest}" - local DISTRO_ID DISTRO_CODENAME - DISTRO_ID=$(awk -F= '/^ID=/{print $2}' /etc/os-release | tr -d '"') - DISTRO_CODENAME=$(awk -F= '/^VERSION_CODENAME=/{print $2}' /etc/os-release) - - # Ensure non-interactive mode for all apt operations - export DEBIAN_FRONTEND=noninteractive - export NEEDRESTART_MODE=a - export NEEDRESTART_SUSPEND=1 - - # Resolve "latest" version - if [[ "$CLICKHOUSE_VERSION" == "latest" ]]; then - CLICKHOUSE_VERSION=$(curl -fsSL --max-time 15 https://packages.clickhouse.com/tgz/stable/ 2>/dev/null | - grep -oP 'clickhouse-common-static-\K[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | - sort -V | tail -n1 || echo "") - - # Fallback to GitHub API if package server failed - if [[ -z "$CLICKHOUSE_VERSION" ]]; then - CLICKHOUSE_VERSION=$(curl -fsSL --max-time 15 https://api.github.com/repos/ClickHouse/ClickHouse/releases/latest 2>/dev/null | - grep -oP '"tag_name":\s*"v\K[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n1 || echo "") - fi - - [[ -z "$CLICKHOUSE_VERSION" ]] && { - msg_error "Could not determine latest ClickHouse version from any source" - return 250 - } - fi - - # Get currently installed version - local CURRENT_VERSION="" - if command -v clickhouse-server >/dev/null 2>&1; then - CURRENT_VERSION=$(clickhouse-server --version 2>/dev/null | grep -oP 'version \K[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n1) - fi - - # Scenario 1: Already at target version - just update packages - if [[ -n "$CURRENT_VERSION" && "$CURRENT_VERSION" == "$CLICKHOUSE_VERSION" ]]; then - msg_info "Update ClickHouse $CLICKHOUSE_VERSION" - ensure_apt_working || return 100 - - # Perform upgrade with retry logic (non-fatal if fails) - upgrade_packages_with_retry "clickhouse-server" "clickhouse-client" || { - msg_warn "ClickHouse package upgrade had issues, continuing with current version" - } - cache_installed_version "clickhouse" "$CLICKHOUSE_VERSION" - msg_ok "Update ClickHouse $CLICKHOUSE_VERSION" - return 0 - fi - - # Scenario 2: Different version - clean upgrade - if [[ -n "$CURRENT_VERSION" && "$CURRENT_VERSION" != "$CLICKHOUSE_VERSION" ]]; then - msg_info "Upgrade ClickHouse from $CURRENT_VERSION to $CLICKHOUSE_VERSION" - stop_all_services "clickhouse-server" - remove_old_tool_version "clickhouse" - else - msg_info "Setup ClickHouse $CLICKHOUSE_VERSION" - fi - - ensure_dependencies apt-transport-https ca-certificates dirmngr gnupg - - # Prepare repository (cleanup + validation) - prepare_repository_setup "clickhouse" || { - msg_error "Failed to prepare ClickHouse repository" - return 100 - } - - # Setup repository (ClickHouse uses 'stable' suite) - setup_deb822_repo \ - "clickhouse" \ - "https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key" \ - "https://packages.clickhouse.com/deb" \ - "stable" \ - "main" - - # Install packages with retry logic - $STD apt update || { - msg_error "APT update failed for ClickHouse repository" - return 100 - } - - install_packages_with_retry "clickhouse-server" "clickhouse-client" || { - msg_error "Failed to install ClickHouse packages" - return 100 - } - - # Verify installation - if ! command -v clickhouse-server >/dev/null 2>&1; then - msg_error "ClickHouse installation completed but clickhouse-server command not found" - return 127 - fi - - # Setup data directory - mkdir -p /var/lib/clickhouse - if id clickhouse >/dev/null 2>&1; then - chown -R clickhouse:clickhouse /var/lib/clickhouse - fi - - # Enable and start service - $STD systemctl enable clickhouse-server || { - msg_warn "Failed to enable clickhouse-server service" - } - safe_service_restart clickhouse-server || true - - cache_installed_version "clickhouse" "$CLICKHOUSE_VERSION" - msg_ok "Setup ClickHouse $CLICKHOUSE_VERSION" -} - # ------------------------------------------------------------------------------ # Installs Rust toolchain and optional global crates via cargo. # @@ -7775,7 +8230,7 @@ function setup_clickhouse() { # RUST_CRATES - Comma-separated list of crates (e.g. "cargo-edit,wasm-pack@0.12.1") # ------------------------------------------------------------------------------ -function setup_rust() { +setup_rust() { local RUST_TOOLCHAIN="${RUST_TOOLCHAIN:-stable}" local RUST_CRATES="${RUST_CRATES:-}" local CARGO_BIN="${HOME}/.cargo/bin" @@ -7791,6 +8246,7 @@ function setup_rust() { msg_info "Setup Rust ($RUST_TOOLCHAIN)" curl -fsSL https://sh.rustup.rs | $STD sh -s -- -y --default-toolchain "$RUST_TOOLCHAIN" || { msg_error "Failed to install Rust" + msg_error "Hint: Check connectivity to sh.rustup.rs and static.rust-lang.org" return 7 } export PATH="$CARGO_BIN:$PATH" @@ -7919,7 +8375,7 @@ function setup_rust() { # - Optionally installs a specific Python version via uv # ------------------------------------------------------------------------------ -function setup_uv() { +setup_uv() { local UV_BIN="/usr/local/bin/uv" local UVX_BIN="/usr/local/bin/uvx" local TMP_DIR=$(mktemp -d) @@ -7960,22 +8416,11 @@ function setup_uv() { ensure_dependencies jq # Fetch latest version - local releases_json - releases_json=$(curl -fsSL --max-time 15 \ - "https://api.github.com/repos/astral-sh/uv/releases/latest" 2>/dev/null || echo "") - - if [[ -z "$releases_json" ]]; then + local LATEST_VERSION + LATEST_VERSION=$(get_latest_github_release "astral-sh/uv") || { msg_error "Could not fetch latest uv version from GitHub API" return 7 - fi - - local LATEST_VERSION - LATEST_VERSION=$(echo "$releases_json" | jq -r '.tag_name' 2>/dev/null | sed 's/^v//') - - if [[ -z "$LATEST_VERSION" ]]; then - msg_error "Could not parse uv version from GitHub API response" - return 250 - fi + } # Get currently installed version local INSTALLED_VERSION="" @@ -8008,6 +8453,7 @@ function setup_uv() { if ! curl_with_retry "$UV_URL" "$TMP_DIR/uv.tar.gz"; then msg_error "Failed to download uv from $UV_URL" + msg_error "Hint: GitHub Releases — check connectivity or set GITHUB_TOKEN to avoid rate-limiting" return 7 fi @@ -8086,7 +8532,7 @@ EOF # - Updates if outdated or wrong implementation # ------------------------------------------------------------------------------ -function setup_yq() { +setup_yq() { local TMP_DIR=$(mktemp -d) local BINARY_PATH="/usr/local/bin/yq" local GITHUB_REPO="mikefarah/yq" @@ -8102,22 +8548,11 @@ function setup_yq() { fi local LATEST_VERSION - local releases_json - releases_json=$(curl -fsSL --max-time 15 "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" 2>/dev/null || echo "") - - if [[ -z "$releases_json" ]]; then + LATEST_VERSION=$(get_latest_github_release "${GITHUB_REPO}") || { msg_error "Could not fetch latest yq version from GitHub API" rm -rf "$TMP_DIR" return 250 - fi - - LATEST_VERSION=$(echo "$releases_json" | jq -r '.tag_name' 2>/dev/null | sed 's/^v//' || echo "") - - if [[ -z "$LATEST_VERSION" ]]; then - msg_error "Could not parse yq version from GitHub API response" - rm -rf "$TMP_DIR" - return 250 - fi + } # Get currently installed version local INSTALLED_VERSION="" @@ -8141,6 +8576,7 @@ function setup_yq() { if ! curl_with_retry "https://github.com/${GITHUB_REPO}/releases/download/v${LATEST_VERSION}/yq_linux_amd64" "$TMP_DIR/yq"; then msg_error "Failed to download yq" + msg_error "Hint: Check connectivity to github.com/${GITHUB_REPO} — set GITHUB_TOKEN to avoid rate-limiting" rm -rf "$TMP_DIR" return 250 fi @@ -8161,274 +8597,6 @@ function setup_yq() { msg_ok "Setup yq $FINAL_VERSION" } -# ------------------------------------------------------------------------------ -# Docker Engine Installation and Management (All-In-One) -# -# Description: -# - By default uses distro repository (docker.io) for stability -# - Optionally uses official Docker repository for latest features -# - Detects and migrates old Docker installations -# - Optional: Installs/Updates Portainer CE -# - Updates running containers interactively -# - Cleans up legacy repository files -# -# Usage: -# setup_docker # Uses distro package (recommended) -# USE_DOCKER_REPO=true setup_docker # Uses official Docker repo -# DOCKER_PORTAINER="true" setup_docker -# DOCKER_LOG_DRIVER="json-file" setup_docker -# -# Variables: -# USE_DOCKER_REPO - Set to "true" to use official Docker repository -# (default: false, uses distro docker.io package) -# DOCKER_PORTAINER - Install Portainer CE (optional, "true" to enable) -# DOCKER_LOG_DRIVER - Log driver (optional, default: "journald") -# DOCKER_SKIP_UPDATES - Skip container update check (optional, "true" to skip) -# -# Features: -# - Uses stable distro packages by default -# - Migrates from get.docker.com to repository-based installation -# - Updates Docker Engine if newer version available -# - Interactive container update with multi-select -# - Portainer installation and update support -# ------------------------------------------------------------------------------ -function setup_docker() { - local docker_installed=false - local portainer_installed=false - local USE_DOCKER_REPO="${USE_DOCKER_REPO:-false}" - - # Check if Docker is already installed - if command -v docker &>/dev/null; then - docker_installed=true - DOCKER_CURRENT_VERSION=$(docker --version | grep -oP '\d+\.\d+\.\d+' | head -1) - msg_info "Docker $DOCKER_CURRENT_VERSION detected" - fi - - # Check if Portainer is running - if docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^portainer$'; then - portainer_installed=true - msg_info "Portainer container detected" - fi - - # Scenario 1: Use distro repository (default, most stable) - if [[ "$USE_DOCKER_REPO" != "true" && "$USE_DOCKER_REPO" != "TRUE" && "$USE_DOCKER_REPO" != "1" ]]; then - - # Install or upgrade Docker from distro repo - if [ "$docker_installed" = true ]; then - msg_info "Checking for Docker updates (distro package)" - ensure_apt_working || return 100 - upgrade_packages_with_retry "docker.io" "docker-compose" || true - DOCKER_CURRENT_VERSION=$(docker --version | grep -oP '\d+\.\d+\.\d+' | head -1) - msg_ok "Docker is up-to-date ($DOCKER_CURRENT_VERSION)" - else - msg_info "Installing Docker (distro package)" - ensure_apt_working || return 100 - - # Install docker.io and docker-compose from distro - if ! install_packages_with_retry "docker.io"; then - msg_error "Failed to install docker.io from distro repository" - return 100 - fi - # docker-compose is optional - $STD apt install -y docker-compose 2>/dev/null || true - - DOCKER_CURRENT_VERSION=$(docker --version | grep -oP '\d+\.\d+\.\d+' | head -1) - msg_ok "Installed Docker $DOCKER_CURRENT_VERSION (distro package)" - fi - - # Configure daemon.json - local log_driver="${DOCKER_LOG_DRIVER:-journald}" - mkdir -p /etc/docker - if [ ! -f /etc/docker/daemon.json ]; then - cat </etc/docker/daemon.json -{ - "log-driver": "$log_driver" -} -EOF - fi - - # Enable and start Docker - systemctl enable -q --now docker - - # Continue to Portainer section below - else - # Scenario 2: Use official Docker repository (USE_DOCKER_REPO=true) - - # Cleanup old repository configurations - if [ -f /etc/apt/sources.list.d/docker.list ]; then - msg_info "Migrating from old Docker repository format" - rm -f /etc/apt/sources.list.d/docker.list - rm -f /etc/apt/keyrings/docker.asc - fi - - # Setup/Update Docker repository - msg_info "Setting up Docker Repository" - setup_deb822_repo \ - "docker" \ - "https://download.docker.com/linux/$(get_os_info id)/gpg" \ - "https://download.docker.com/linux/$(get_os_info id)" \ - "$(get_os_info codename)" \ - "stable" \ - "$(dpkg --print-architecture)" - - # Install or upgrade Docker - if [ "$docker_installed" = true ]; then - msg_info "Checking for Docker updates" - DOCKER_LATEST_VERSION=$(apt-cache policy docker-ce | grep Candidate | awk '{print $2}' 2>/dev/null | cut -d':' -f2 | cut -d'-' -f1 || echo '') - - if [ "$DOCKER_CURRENT_VERSION" != "$DOCKER_LATEST_VERSION" ]; then - msg_info "Updating Docker $DOCKER_CURRENT_VERSION → $DOCKER_LATEST_VERSION" - $STD apt install -y --only-upgrade \ - docker-ce \ - docker-ce-cli \ - containerd.io \ - docker-buildx-plugin \ - docker-compose-plugin || { - msg_error "Failed to update Docker packages" - return 100 - } - msg_ok "Updated Docker to $DOCKER_LATEST_VERSION" - else - msg_ok "Docker is up-to-date ($DOCKER_CURRENT_VERSION)" - fi - else - msg_info "Installing Docker" - $STD apt install -y \ - docker-ce \ - docker-ce-cli \ - containerd.io \ - docker-buildx-plugin \ - docker-compose-plugin || { - msg_error "Failed to install Docker packages" - return 100 - } - - DOCKER_CURRENT_VERSION=$(docker --version | grep -oP '\d+\.\d+\.\d+' | head -1) - msg_ok "Installed Docker $DOCKER_CURRENT_VERSION" - fi - - # Configure daemon.json - local log_driver="${DOCKER_LOG_DRIVER:-journald}" - mkdir -p /etc/docker - if [ ! -f /etc/docker/daemon.json ]; then - cat </etc/docker/daemon.json -{ - "log-driver": "$log_driver" -} -EOF - fi - - # Enable and start Docker - systemctl enable -q --now docker - fi - - # Portainer Management (common for both modes) - if [[ "${DOCKER_PORTAINER:-}" == "true" ]]; then - if [ "$portainer_installed" = true ]; then - msg_info "Checking for Portainer updates" - PORTAINER_CURRENT=$(docker inspect portainer --format='{{.Config.Image}}' 2>/dev/null | cut -d':' -f2) - PORTAINER_LATEST=$(curl -fsSL https://registry.hub.docker.com/v2/repositories/portainer/portainer-ce/tags?page_size=100 | grep -oP '"name":"\K[0-9]+\.[0-9]+\.[0-9]+"' | head -1 | tr -d '"') - - if [ "$PORTAINER_CURRENT" != "$PORTAINER_LATEST" ]; then - read -r -p "${TAB3}Update Portainer $PORTAINER_CURRENT → $PORTAINER_LATEST? " prompt - if [[ ${prompt,,} =~ ^(y|yes)$ ]]; then - msg_info "Updating Portainer" - docker stop portainer - docker rm portainer - docker pull portainer/portainer-ce:latest - docker run -d \ - -p 9000:9000 \ - -p 9443:9443 \ - --name=portainer \ - --restart=always \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v portainer_data:/data \ - portainer/portainer-ce:latest - msg_ok "Updated Portainer to $PORTAINER_LATEST" - fi - else - msg_ok "Portainer is up-to-date ($PORTAINER_CURRENT)" - fi - else - msg_info "Installing Portainer" - docker volume create portainer_data - docker run -d \ - -p 9000:9000 \ - -p 9443:9443 \ - --name=portainer \ - --restart=always \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v portainer_data:/data \ - portainer/portainer-ce:latest - - LOCAL_IP=$(hostname -I | awk '{print $1}') - msg_ok "Installed Portainer (http://${LOCAL_IP}:9000)" - fi - fi - - # Interactive Container Update Check - if [[ "${DOCKER_SKIP_UPDATES:-}" != "true" ]] && [ "$docker_installed" = true ]; then - msg_info "Checking for container updates" - - # Get list of running containers with update status - local containers_with_updates=() - local container_info=() - local index=1 - - while IFS= read -r container; do - local name=$(echo "$container" | awk '{print $1}') - local image=$(echo "$container" | awk '{print $2}') - local current_digest=$(docker inspect "$name" --format='{{.Image}}' 2>/dev/null | cut -d':' -f2 | cut -c1-12) - - # Pull latest image digest - docker pull "$image" >/dev/null 2>&1 - local latest_digest=$(docker inspect "$image" --format='{{.Id}}' 2>/dev/null | cut -d':' -f2 | cut -c1-12) - - if [ "$current_digest" != "$latest_digest" ]; then - containers_with_updates+=("$name") - container_info+=("${index}) ${name} (${image})") - ((index++)) - fi - done < <(docker ps --format '{{.Names}} {{.Image}}') - - if [ ${#containers_with_updates[@]} -gt 0 ]; then - echo "" - echo "${TAB3}Container updates available:" - for info in "${container_info[@]}"; do - echo "${TAB3} $info" - done - echo "" - read -r -p "${TAB3}Select containers to update (e.g., 1,3,5 or 'all' or 'none'): " selection - - if [[ ${selection,,} == "all" ]]; then - for container in "${containers_with_updates[@]}"; do - msg_info "Updating container: $container" - docker stop "$container" - docker rm "$container" - # Note: This requires the original docker run command - best to recreate via compose - msg_ok "Stopped and removed $container (please recreate with updated image)" - done - elif [[ ${selection,,} != "none" ]]; then - IFS=',' read -ra SELECTED <<<"$selection" - for num in "${SELECTED[@]}"; do - num=$(echo "$num" | xargs) # trim whitespace - if [[ "$num" =~ ^[0-9]+$ ]] && [ "$num" -ge 1 ] && [ "$num" -le "${#containers_with_updates[@]}" ]; then - container="${containers_with_updates[$((num - 1))]}" - msg_info "Updating container: $container" - docker stop "$container" - docker rm "$container" - msg_ok "Stopped and removed $container (please recreate with updated image)" - fi - done - fi - else - msg_ok "All containers are up-to-date" - fi - fi - - msg_ok "Docker setup completed" -} - # ------------------------------------------------------------------------------ # Fetch and deploy from URL # Downloads an archive (zip, tar.gz, or .deb) from a URL and extracts/installs it @@ -8443,7 +8611,7 @@ EOF # fetch_and_deploy_from_url "https://example.com/app.zip" "/opt/myapp" # fetch_and_deploy_from_url "https://example.com/package.deb" "" # ------------------------------------------------------------------------------ -function fetch_and_deploy_from_url() { +fetch_and_deploy_from_url() { local url="$1" local directory="${2:-}" @@ -8496,7 +8664,7 @@ function fetch_and_deploy_from_url() { chmod 644 "$tmpdir/$filename" $STD apt install -y "$tmpdir/$filename" || { $STD dpkg -i "$tmpdir/$filename" || { - msg_error "Both apt and dpkg installation failed" + _diagnose_deb_failure "$tmpdir/$filename" rm -rf "$tmpdir" return 100 } @@ -8597,3 +8765,829 @@ EOF $STD apt update return 0 } + +# ------------------------------------------------------------------------------ +# Get latest GitLab release version. +# Usage: get_latest_gitlab_release "owner/repo" [strip_v] +# ------------------------------------------------------------------------------ +get_latest_gitlab_release() { + local repo="$1" + local strip_v="${2:-true}" + + local repo_encoded + repo_encoded=$(printf '%s' "$repo" | sed 's|/|%2F|g') + + local header=() + [[ -n "${GITLAB_TOKEN:-}" ]] && header=(-H "PRIVATE-TOKEN: $GITLAB_TOKEN") + + local temp_file + temp_file=$(mktemp) + + local http_code + http_code=$(curl --connect-timeout 10 --max-time 30 -sSL \ + -w "%{http_code}" -o "$temp_file" \ + "${header[@]}" \ + "https://gitlab.com/api/v4/projects/$repo_encoded/releases?per_page=1&order_by=released_at&sort=desc" 2>/dev/null) || true + + if [[ "$http_code" != "200" ]]; then + rm -f "$temp_file" + msg_warn "GitLab API call failed for ${repo} (HTTP ${http_code})" + return 22 + fi + + local version + version=$(jq -r '.[0].tag_name // empty' "$temp_file") + rm -f "$temp_file" + + if [[ -z "$version" ]]; then + msg_error "Could not determine latest version for ${repo}" + return 250 + fi + + if [[ "$strip_v" == "true" ]]; then + [[ "$version" =~ ^v[0-9] ]] && version="${version:1}" + fi + + echo "$version" +} + +# ------------------------------------------------------------------------------ +# Checks for new GitLab release (latest tag). +# +# Description: +# - Queries the GitLab API for the latest release tag +# - Compares it to a local cached version (~/.) +# - If newer, sets global CHECK_UPDATE_RELEASE and returns 0 +# +# Usage: +# if check_for_gl_release "myapp" "owner/repo" [optional] "v1.2.3"; then +# # trigger update... +# fi +# exit 0 +# } (end of update_script not from the function) +# +# Notes: +# - Requires `jq` (auto-installed if missing) +# - Supports GITLAB_TOKEN env var for private/rate-limited repos +# - Does not modify anything, only checks version state +# ------------------------------------------------------------------------------ +check_for_gl_release() { + local app="$1" + local source="$2" + local pinned_version_in="${3:-}" # optional + local pin_reason="${4:-}" # optional reason shown to user + local app_lc="${app,,}" + local current_file="$HOME/.${app_lc}" + + msg_info "Checking for update: ${app}" + + # DNS check + if ! getent hosts gitlab.com >/dev/null 2>&1; then + msg_error "Network error: cannot resolve gitlab.com" + return 6 + fi + + ensure_dependencies jq + + local repo_encoded + repo_encoded=$(printf '%s' "$source" | sed 's|/|%2F|g') + + local header=() + [[ -n "${GITLAB_TOKEN:-}" ]] && header=(-H "PRIVATE-TOKEN: $GITLAB_TOKEN") + + local releases_json="" http_code="" + + # For pinned versions, try to fetch the specific release tag first + if [[ -n "$pinned_version_in" ]]; then + local pinned_encoded="${pinned_version_in//\//%2F}" + http_code=$(curl -sSL --max-time 20 -w "%{http_code}" -o /tmp/gl_check.json \ + "${header[@]}" \ + "https://gitlab.com/api/v4/projects/$repo_encoded/releases/$pinned_encoded" 2>/dev/null) || true + if [[ "$http_code" == "200" ]] && [[ -s /tmp/gl_check.json ]]; then + releases_json="[$(/dev/null) || true + + if [[ "$http_code" == "200" ]] && [[ -s /tmp/gl_check.json ]]; then + releases_json=$(/dev/null) + if ((${#legacy_files[@]} == 1)); then + current="$(<"${legacy_files[0]}")" + echo "${current#v}" >"$current_file" + rm -f "${legacy_files[0]}" + fi + fi + if [[ "$current" =~ ^v[0-9] ]]; then + current="${current:1}" + fi + + # Pinned version handling + if [[ -n "$pinned_version_in" ]]; then + local pin_clean + if [[ "$pinned_version_in" =~ ^v[0-9] ]]; then + pin_clean="${pinned_version_in:1}" + else + pin_clean="$pinned_version_in" + fi + local match_raw="" + for i in "${!clean_tags[@]}"; do + if [[ "${clean_tags[$i]}" == "$pin_clean" ]]; then + match_raw="${raw_tags[$i]}" + break + fi + done + + if [[ -z "$match_raw" ]]; then + msg_error "Pinned version ${pinned_version_in} not found upstream" + return 250 + fi + + if [[ "$current" != "$pin_clean" ]]; then + CHECK_UPDATE_RELEASE="$match_raw" + msg_ok "Update available: ${app} ${current:-not installed} → ${pin_clean}" + return 0 + fi + + if [[ -n "$pin_reason" ]]; then + msg_ok "No update available: ${app} (${current}) - update held back: ${pin_reason}" + else + msg_ok "No update available: ${app} (${current}) - update temporarily held back due to issues with newer releases" + fi + return 1 + fi + + # No pinning → use latest + if [[ -z "$current" || "$current" != "$latest_clean" ]]; then + CHECK_UPDATE_RELEASE="$latest_raw" + msg_ok "Update available: ${app} ${current:-not installed} → ${latest_clean}" + return 0 + fi + + msg_ok "No update available: ${app} (${latest_clean})" + return 1 +} + +# ------------------------------------------------------------------------------ +# Scan older GitLab releases for a matching asset (fallback helper). +# +# Description: +# When the latest release does not contain the expected asset +# (e.g. .deb for the current arch, or a custom pattern), walks back +# through up to 15 recent releases and returns the first release JSON +# that has a matching asset. Used internally by fetch_and_deploy_gl_release. +# +# Usage (internal): +# _gl_scan_older_releases "owner/repo" "owner%2Frepo" "https://gitlab.com" \ +# "binary|prebuild|singlefile" "$asset_pattern" "$skip_tag" +# +# Returns: +# - stdout: JSON of the matching release (single object) on success +# - 0 on success, 22 on API error, 250 if no match found +# ------------------------------------------------------------------------------ +_gl_scan_older_releases() { + local repo="$1" + local repo_encoded="$2" + local base_url="${3:-https://gitlab.com}" + local mode="$4" + local asset_pattern="$5" + local skip_tag="$6" + + local header=() + [[ -n "${GITLAB_TOKEN:-}" ]] && header=(-H "PRIVATE-TOKEN: $GITLAB_TOKEN") + + local releases_list + releases_list=$(curl --connect-timeout 10 --max-time 30 -fsSL \ + "${header[@]}" \ + "${base_url}/api/v4/projects/${repo_encoded}/releases?per_page=15&order_by=released_at&sort=desc" 2>/dev/null) || { + msg_warn "Failed to fetch older releases for ${repo}" + return 22 + } + + local count + count=$(echo "$releases_list" | jq 'length' 2>/dev/null || echo 0) + [[ "$count" -eq 0 ]] && return 250 + + for ((i = 0; i < count; i++)); do + local rel_tag + rel_tag=$(echo "$releases_list" | jq -r ".[$i].tag_name") + + # Skip the tag we already checked + [[ "$rel_tag" == "$skip_tag" ]] && continue + + # Asset URLs for this release (direct_asset_url preferred, fallback to url) + local asset_urls + asset_urls=$(echo "$releases_list" | jq -r ".[$i].assets.links // [] | .[] | .direct_asset_url // .url") + [[ -z "$asset_urls" ]] && continue + + local has_match=false + + if [[ "$mode" == "binary" ]]; then + local arch + arch=$(dpkg --print-architecture 2>/dev/null || uname -m) + [[ "$arch" == "x86_64" ]] && arch="amd64" + [[ "$arch" == "aarch64" ]] && arch="arm64" + + # Check with explicit pattern first, then arch heuristic, then any .deb + if [[ -n "$asset_pattern" ]]; then + while read -r u; do + case "${u##*/}" in $asset_pattern) + has_match=true + break + ;; + esac + done <<<"$asset_urls" + fi + if [[ "$has_match" != "true" ]]; then + echo "$asset_urls" | grep -qE "($arch|amd64|x86_64|aarch64|arm64).*\.deb$" && has_match=true + fi + if [[ "$has_match" != "true" ]]; then + echo "$asset_urls" | grep -qE '\.deb$' && has_match=true + fi + + elif [[ "$mode" == "prebuild" || "$mode" == "singlefile" ]]; then + while read -r u; do + case "${u##*/}" in $asset_pattern) + has_match=true + break + ;; + esac + done <<<"$asset_urls" + fi + + if [[ "$has_match" == "true" ]]; then + local use_fallback="y" + if [[ -t 0 ]]; then + msg_warn "Release ${skip_tag} has no matching asset. Previous release ${rel_tag} has a compatible asset." + read -rp "Use version ${rel_tag} instead? [Y/n] (auto-yes in 60s): " -t 60 use_fallback || use_fallback="y" + use_fallback="${use_fallback:-y}" + fi + + if [[ "${use_fallback,,}" == "y" || "${use_fallback,,}" == "yes" ]]; then + echo "$releases_list" | jq ".[$i]" + return 0 + else + return 250 + fi + fi + done + + return 250 +} + +fetch_and_deploy_gl_release() { + local app="$1" + local repo="$2" + local mode="${3:-tarball}" + local version="${var_appversion:-${4:-latest}}" + local target="${5:-/opt/$app}" + local asset_pattern="${6:-}" + + if [[ -z "$app" ]]; then + app="${repo##*/}" + if [[ -z "$app" ]]; then + msg_error "fetch_and_deploy_gl_release requires app name or valid repo" + return 1 + fi + fi + + local app_lc=$(echo "${app,,}" | tr -d ' ') + local version_file="$HOME/.${app_lc}" + + local api_timeout="--connect-timeout 10 --max-time 60" + local download_timeout="--connect-timeout 15 --max-time 900" + + local current_version="" + [[ -f "$version_file" ]] && current_version=$(<"$version_file") + + ensure_dependencies jq + + local repo_encoded + repo_encoded=$(printf '%s' "$repo" | sed 's|/|%2F|g') + + local api_base="https://gitlab.com/api/v4/projects/$repo_encoded/releases" + local api_url + if [[ "$version" != "latest" ]]; then + api_url="$api_base/$version" + else + api_url="$api_base?per_page=1&order_by=released_at&sort=desc" + fi + + local header=() + [[ -n "${GITLAB_TOKEN:-}" ]] && header=(-H "PRIVATE-TOKEN: $GITLAB_TOKEN") + + local max_retries=3 retry_delay=2 attempt=1 success=false http_code + + while ((attempt <= max_retries)); do + http_code=$(curl $api_timeout -sSL -w "%{http_code}" -o /tmp/gl_rel.json "${header[@]}" "$api_url" 2>/dev/null) || true + if [[ "$http_code" == "200" ]]; then + success=true + break + elif [[ "$http_code" == "429" ]]; then + if ((attempt < max_retries)); then + msg_warn "GitLab API rate limit hit, retrying in ${retry_delay}s... (attempt $attempt/$max_retries)" + sleep "$retry_delay" + retry_delay=$((retry_delay * 2)) + fi + else + sleep "$retry_delay" + fi + ((attempt++)) + done + + if ! $success; then + if [[ "$http_code" == "401" ]]; then + msg_error "GitLab API authentication failed (HTTP 401)." + if [[ -n "${GITLAB_TOKEN:-}" ]]; then + msg_error "Your GITLAB_TOKEN appears to be invalid or expired." + else + msg_error "The repository may require authentication. Try: export GITLAB_TOKEN=\"glpat-your_token\"" + fi + elif [[ "$http_code" == "404" ]]; then + msg_error "GitLab project or release not found (HTTP 404)." + msg_error "Ensure '$repo' is correct and the project is accessible." + elif [[ "$http_code" == "429" ]]; then + msg_error "GitLab API rate limit exceeded (HTTP 429)." + msg_error "To increase the limit, export a GitLab token before running the script:" + msg_error " export GITLAB_TOKEN=\"glpat-your_token_here\"" + elif [[ "$http_code" == "000" || -z "$http_code" ]]; then + msg_error "GitLab API connection failed (no response)." + msg_error "Check your network/DNS: curl -sSL https://gitlab.com/api/v4/version" + else + msg_error "Failed to fetch release metadata (HTTP $http_code)" + fi + return 1 + fi + + local json tag_name + json=$(/dev/null || uname -m) + [[ "$arch" == "x86_64" ]] && arch="amd64" + [[ "$arch" == "aarch64" ]] && arch="arm64" + + local assets url_match="" + assets=$(_gl_asset_urls "$json") + + if [[ -n "$asset_pattern" ]]; then + for u in $assets; do + case "${u##*/}" in + $asset_pattern) + url_match="$u" + break + ;; + esac + done + fi + + if [[ -z "$url_match" ]]; then + for u in $assets; do + if [[ "$u" =~ ($arch|amd64|x86_64|aarch64|arm64).*\.deb$ ]]; then + url_match="$u" + break + fi + done + fi + + if [[ -z "$url_match" ]]; then + for u in $assets; do + [[ "$u" =~ \.deb$ ]] && url_match="$u" && break + done + fi + + if [[ -z "$url_match" ]]; then + local fallback_json + if fallback_json=$(_gl_scan_older_releases "$repo" "$repo_encoded" "https://gitlab.com" "binary" "$asset_pattern" "$tag_name"); then + json="$fallback_json" + tag_name=$(echo "$json" | jq -r '.tag_name // empty') + [[ "$tag_name" =~ ^v[0-9] ]] && version="${tag_name:1}" || version="$tag_name" + msg_info "Fetching GitLab release: $app ($version)" + assets=$(_gl_asset_urls "$json") + if [[ -n "$asset_pattern" ]]; then + for u in $assets; do + case "${u##*/}" in $asset_pattern) + url_match="$u" + break + ;; + esac + done + fi + if [[ -z "$url_match" ]]; then + for u in $assets; do + [[ "$u" =~ ($arch|amd64|x86_64|aarch64|arm64).*\.deb$ ]] && url_match="$u" && break + done + fi + if [[ -z "$url_match" ]]; then + for u in $assets; do + [[ "$u" =~ \.deb$ ]] && url_match="$u" && break + done + fi + fi + fi + + if [[ -z "$url_match" ]]; then + msg_error "No suitable .deb asset found for $app" + rm -rf "$tmpdir" + return 1 + fi + + filename="${url_match##*/}" + curl $download_timeout -fsSL "${header[@]}" -o "$tmpdir/$filename" "$url_match" || { + msg_error "Download failed: $url_match" + rm -rf "$tmpdir" + return 1 + } + + chmod 644 "$tmpdir/$filename" + local dpkg_opts="" + [[ "${DPKG_FORCE_CONFOLD:-}" == "1" ]] && dpkg_opts="-o Dpkg::Options::=--force-confold" + [[ "${DPKG_FORCE_CONFNEW:-}" == "1" ]] && dpkg_opts="-o Dpkg::Options::=--force-confnew" + DEBIAN_FRONTEND=noninteractive SYSTEMD_OFFLINE=1 $STD apt install -y $dpkg_opts "$tmpdir/$filename" || { + SYSTEMD_OFFLINE=1 $STD dpkg -i "$tmpdir/$filename" || { + _diagnose_deb_failure "$tmpdir/$filename" + rm -rf "$tmpdir" + return 1 + } + } + + ### Prebuild Mode ### + elif [[ "$mode" == "prebuild" ]]; then + local pattern="${6%\"}" + pattern="${pattern#\"}" + [[ -z "$pattern" ]] && { + msg_error "Mode 'prebuild' requires 6th parameter (asset filename pattern)" + rm -rf "$tmpdir" + return 1 + } + + local asset_url="" + for u in $(_gl_asset_urls "$json"); do + filename_candidate="${u##*/}" + case "$filename_candidate" in + $pattern) + asset_url="$u" + break + ;; + esac + done + + if [[ -z "$asset_url" ]]; then + local fallback_json + if fallback_json=$(_gl_scan_older_releases "$repo" "$repo_encoded" "https://gitlab.com" "prebuild" "$pattern" "$tag_name"); then + json="$fallback_json" + tag_name=$(echo "$json" | jq -r '.tag_name // empty') + [[ "$tag_name" =~ ^v[0-9] ]] && version="${tag_name:1}" || version="$tag_name" + msg_info "Fetching GitLab release: $app ($version)" + for u in $(_gl_asset_urls "$json"); do + filename_candidate="${u##*/}" + case "$filename_candidate" in $pattern) + asset_url="$u" + break + ;; + esac + done + fi + fi + + [[ -z "$asset_url" ]] && { + msg_error "No asset matching '$pattern' found" + rm -rf "$tmpdir" + return 1 + } + + filename="${asset_url##*/}" + curl $download_timeout -fsSL "${header[@]}" -o "$tmpdir/$filename" "$asset_url" || { + msg_error "Download failed: $asset_url" + rm -rf "$tmpdir" + return 1 + } + + local unpack_tmp + unpack_tmp=$(mktemp -d) + mkdir -p "$target" + if [[ "${CLEAN_INSTALL:-0}" == "1" ]]; then + rm -rf "${target:?}/"* + fi + + if [[ "$filename" == *.zip ]]; then + ensure_dependencies unzip + unzip -q "$tmpdir/$filename" -d "$unpack_tmp" || { + msg_error "Failed to extract ZIP archive" + rm -rf "$tmpdir" "$unpack_tmp" + return 1 + } + elif [[ "$filename" == *.tar.* || "$filename" == *.tgz || "$filename" == *.txz ]]; then + tar --no-same-owner -xf "$tmpdir/$filename" -C "$unpack_tmp" || { + msg_error "Failed to extract TAR archive" + rm -rf "$tmpdir" "$unpack_tmp" + return 1 + } + else + msg_error "Unsupported archive format: $filename" + rm -rf "$tmpdir" "$unpack_tmp" + return 1 + fi + + local top_entries inner_dir + top_entries=$(find "$unpack_tmp" -mindepth 1 -maxdepth 1) + if [[ "$(echo "$top_entries" | wc -l)" -eq 1 && -d "$top_entries" ]]; then + inner_dir="$top_entries" + shopt -s dotglob nullglob + if compgen -G "$inner_dir/*" >/dev/null; then + cp -r "$inner_dir"/* "$target/" || { + msg_error "Failed to copy contents from $inner_dir to $target" + rm -rf "$tmpdir" "$unpack_tmp" + return 1 + } + else + msg_error "Inner directory is empty: $inner_dir" + rm -rf "$tmpdir" "$unpack_tmp" + return 1 + fi + shopt -u dotglob nullglob + else + shopt -s dotglob nullglob + if compgen -G "$unpack_tmp/*" >/dev/null; then + cp -r "$unpack_tmp"/* "$target/" || { + msg_error "Failed to copy contents to $target" + rm -rf "$tmpdir" "$unpack_tmp" + return 1 + } + else + msg_error "Unpacked archive is empty" + rm -rf "$tmpdir" "$unpack_tmp" + return 1 + fi + shopt -u dotglob nullglob + fi + + ### Singlefile Mode ### + elif [[ "$mode" == "singlefile" ]]; then + local pattern="${6%\"}" + pattern="${pattern#\"}" + [[ -z "$pattern" ]] && { + msg_error "Mode 'singlefile' requires 6th parameter (asset filename pattern)" + rm -rf "$tmpdir" + return 1 + } + + local asset_url="" + for u in $(_gl_asset_urls "$json"); do + filename_candidate="${u##*/}" + case "$filename_candidate" in + $pattern) + asset_url="$u" + break + ;; + esac + done + + if [[ -z "$asset_url" ]]; then + local fallback_json + if fallback_json=$(_gl_scan_older_releases "$repo" "$repo_encoded" "https://gitlab.com" "singlefile" "$pattern" "$tag_name"); then + json="$fallback_json" + tag_name=$(echo "$json" | jq -r '.tag_name // empty') + [[ "$tag_name" =~ ^v[0-9] ]] && version="${tag_name:1}" || version="$tag_name" + msg_info "Fetching GitLab release: $app ($version)" + for u in $(_gl_asset_urls "$json"); do + filename_candidate="${u##*/}" + case "$filename_candidate" in $pattern) + asset_url="$u" + break + ;; + esac + done + fi + fi + + [[ -z "$asset_url" ]] && { + msg_error "No asset matching '$pattern' found" + rm -rf "$tmpdir" + return 1 + } + + filename="${asset_url##*/}" + mkdir -p "$target" + + local use_filename="${USE_ORIGINAL_FILENAME:-false}" + local target_file="$app" + [[ "$use_filename" == "true" ]] && target_file="$filename" + + curl $download_timeout -fsSL "${header[@]}" -o "$target/$target_file" "$asset_url" || { + msg_error "Download failed: $asset_url" + rm -rf "$tmpdir" + return 1 + } + + if [[ "$target_file" != *.jar && -f "$target/$target_file" ]]; then + chmod +x "$target/$target_file" + fi + + else + msg_error "Unknown mode: $mode" + rm -rf "$tmpdir" + return 1 + fi + + echo "$version" >"$version_file" + msg_ok "Deployed: $app ($version)" + rm -rf "$tmpdir" +} + +# ------------------------------------------------------------------------------ +# Download NLTK data packages directly from GitHub, bypassing Python. +# Avoids CPU-instruction failures (SIGILL) on older hardware lacking AVX. +# +# Usage: +# setup_nltk "averaged_perceptron_tagger_eng" "/nltk_data" +# setup_nltk "snowball_data stopwords punkt_tab" "/usr/share/nltk_data" +# +# Parameters: +# $1 - Space-separated list of NLTK package IDs +# $2 - Target directory (default: /usr/share/nltk_data) +# +# Returns: 0 on success, non-zero if any package failed +# ------------------------------------------------------------------------------ +setup_nltk() { + local packages="${1:?setup_nltk requires at least one package name}" + local target_dir="${2:-/usr/share/nltk_data}" + local NLTK_INDEX_URL="https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml" + local index_xml rc=0 + + ensure_dependencies unzip + + index_xml=$(curl_with_retry "$NLTK_INDEX_URL" "-") || { + msg_error "Failed to fetch NLTK package index" + return 1 + } + + local pkg + for pkg in $packages; do + msg_info "Downloading NLTK: $pkg" + local pkg_line subdir pkg_url do_unzip tmp_zip + + pkg_line=$(echo "$index_xml" | grep "id=\"${pkg}\"" | head -1) + if [[ -z "$pkg_line" ]]; then + msg_error "NLTK package not found in index: $pkg" + rc=1 + continue + fi + + subdir=$(echo "$pkg_line" | grep -oP 'subdir="\K[^"]+') + pkg_url=$(echo "$pkg_line" | grep -oP 'url="\K[^"]+') + do_unzip=$(echo "$pkg_line" | grep -oP 'unzip="\K[^"]+') + + if [[ -z "$subdir" || -z "$pkg_url" ]]; then + msg_error "Could not parse NLTK index entry for: $pkg" + rc=1 + continue + fi + + mkdir -p "${target_dir}/${subdir}" + tmp_zip=$(mktemp --suffix=.zip) + + if CURL_TIMEOUT=120 curl_with_retry "$pkg_url" "$tmp_zip"; then + if [[ "$do_unzip" == "1" ]]; then + $STD unzip -q -o "$tmp_zip" -d "${target_dir}/${subdir}/" + rm -f "$tmp_zip" + else + mv "$tmp_zip" "${target_dir}/${subdir}/${pkg}.zip" + fi + msg_ok "Downloaded NLTK: $pkg" + else + msg_error "Failed to download NLTK package: $pkg" + rm -f "$tmp_zip" + rc=1 + fi + done + + return $rc +} diff --git a/src/server/ssh-service.js b/src/server/ssh-service.js index c02fcca..fefac10 100644 --- a/src/server/ssh-service.js +++ b/src/server/ssh-service.js @@ -12,11 +12,11 @@ class SSHService { */ async testConnection(server) { const { auth_type = 'password' } = server; - + return new Promise((resolve) => { const timeout = 15000; // 15 seconds timeout for login test let resolved = false; - + // Choose authentication method based on auth_type let authPromise; if (auth_type === 'key') { @@ -25,7 +25,7 @@ class SSHService { // Default to password authentication authPromise = this.testWithSshpass(server).catch(() => this.testWithExpect(server)); } - + authPromise.then(result => { if (!resolved) { resolved = true; @@ -45,7 +45,7 @@ class SSHService { }); } }); - + // Set up overall timeout setTimeout(() => { if (!resolved) { @@ -67,15 +67,15 @@ class SSHService { */ async testWithSshpass(server) { const { ip, user, password, ssh_port = 22 } = server; - + if (!password) { throw new Error('Password is required for password authentication'); } - + return new Promise((resolve, reject) => { const timeout = 10000; let resolved = false; - + const sshCommand = spawn('sshpass', [ '-p', password, 'ssh', @@ -102,7 +102,7 @@ class SSHService { let output = ''; let errorOutput = ''; - + sshCommand.stdout.on('data', (data) => { output += data.toString(); }); @@ -115,7 +115,7 @@ class SSHService { if (!resolved) { resolved = true; clearTimeout(timer); - + if (code === 0 && output.includes('SSH_LOGIN_SUCCESS')) { resolve({ success: true, @@ -129,7 +129,7 @@ class SSHService { }); } else { let errorMessage = 'SSH login failed'; - + if (errorOutput.includes('Permission denied') || errorOutput.includes('Authentication failed')) { errorMessage = 'Authentication failed - check username and password'; } else if (errorOutput.includes('Connection refused')) { @@ -164,11 +164,11 @@ class SSHService { */ async testWithExpect(server) { const { ip, user, password, ssh_port = 22 } = server; - + return new Promise((resolve, reject) => { const timeout = 10000; let resolved = false; - + // Pass password via env so it is not embedded in the script (safe for special chars like {, $, "). const expectScript = `#!/usr/bin/expect -f set timeout 10 @@ -208,7 +208,7 @@ expect { let output = ''; let errorOutput = ''; - + expectCommand.stdout.on('data', (data) => { output += data.toString(); }); @@ -221,7 +221,7 @@ expect { if (!resolved) { resolved = true; clearTimeout(timer); - + if (code === 0) { resolve({ success: true, @@ -235,7 +235,7 @@ expect { }); } else { let errorMessage = 'SSH login failed'; - + if (errorOutput.includes('Permission denied') || errorOutput.includes('Authentication failed')) { errorMessage = 'Authentication failed - check username and password'; } else if (errorOutput.includes('Connection refused')) { @@ -271,11 +271,11 @@ expect { */ async testConnectionBasic(server) { const { ip, user, password } = server; - + return new Promise((resolve) => { const timeout = 10000; // 10 seconds timeout let resolved = false; - + // First, test if the SSH port is open using netcat or telnet const portTestCommand = spawn('nc', ['-z', '-w', '5', ip, '22'], { stdio: ['pipe', 'pipe', 'pipe'] @@ -299,7 +299,7 @@ expect { if (!resolved) { resolved = true; clearTimeout(timer); - + if (code === 0) { // Port is open, now try a basic SSH connection test this.testSSHConnection(server).then(resolve).catch(() => { @@ -331,7 +331,7 @@ expect { if (!resolved) { resolved = true; clearTimeout(timer); - + // If netcat is not available, try with telnet this.testWithTelnet(server).then(resolve).catch(() => { resolve({ @@ -355,11 +355,11 @@ expect { */ async testWithTelnet(server) { const { ip } = server; - + return new Promise((resolve) => { const timeout = 5000; let resolved = false; - + const telnetCommand = spawn('timeout', ['5', 'telnet', ip, '22'], { stdio: ['pipe', 'pipe', 'pipe'] }); @@ -377,7 +377,7 @@ expect { }, timeout); let output = ''; - + telnetCommand.stdout.on('data', (data) => { output += data.toString(); }); @@ -390,7 +390,7 @@ expect { if (!resolved) { resolved = true; clearTimeout(timer); - + if (output.includes('Connected') || output.includes('SSH')) { resolve({ success: true, @@ -438,11 +438,11 @@ expect { */ async testSSHConnection(server) { const { ip, user, ssh_port = 22 } = server; - + return new Promise((resolve) => { const timeout = 5000; let resolved = false; - + const sshCommand = spawn('ssh', [ '-p', ssh_port.toString(), '-o', 'ConnectTimeout=5', @@ -471,7 +471,7 @@ expect { }, timeout); let errorOutput = ''; - + sshCommand.stderr.on('data', (data) => { errorOutput += data.toString(); }); @@ -480,7 +480,7 @@ expect { if (!resolved) { resolved = true; clearTimeout(timer); - + // SSH connection was established but authentication failed // This is actually a good sign - it means SSH is working if (errorOutput.includes('Permission denied') || errorOutput.includes('Authentication failed')) { @@ -540,7 +540,7 @@ expect { */ async testWithSSHKey(server) { const { ip, user, ssh_key_path, ssh_key_passphrase, ssh_port = 22 } = server; - + if (!ssh_key_path || !existsSync(ssh_key_path)) { throw new Error('SSH key file not found'); } @@ -548,7 +548,7 @@ expect { return new Promise((resolve, reject) => { const timeout = 10000; let resolved = false; - + try { // Build SSH command const sshArgs = [ @@ -563,7 +563,7 @@ expect { `${user}@${ip}`, 'echo "SSH_LOGIN_SUCCESS"' ]; - + // Use sshpass if passphrase is provided let command, args; if (ssh_key_passphrase) { @@ -573,7 +573,7 @@ expect { command = 'ssh'; args = sshArgs; } - + const sshCommand = spawn(command, args, { stdio: ['pipe', 'pipe', 'pipe'] }); @@ -588,7 +588,7 @@ expect { let output = ''; let errorOutput = ''; - + sshCommand.stdout.on('data', (data) => { output += data.toString(); }); @@ -601,7 +601,7 @@ expect { if (!resolved) { resolved = true; clearTimeout(timer); - + if (code === 0 && output.includes('SSH_LOGIN_SUCCESS')) { resolve({ success: true, @@ -615,7 +615,7 @@ expect { }); } else { let errorMessage = 'SSH key authentication failed'; - + if (errorOutput.includes('Permission denied') || errorOutput.includes('Authentication failed')) { errorMessage = 'SSH key authentication failed - check key and permissions'; } else if (errorOutput.includes('Connection refused')) { @@ -644,7 +644,7 @@ expect { reject(error); } }); - + } catch (error) { if (!resolved) { resolved = true; @@ -681,7 +681,7 @@ expect { }); let errorOutput = ''; - + sshKeygen.stderr.on('data', (data) => { errorOutput += data.toString(); }); @@ -691,15 +691,15 @@ expect { try { // Read the generated private key const privateKey = readFileSync(keyPath, 'utf8'); - + // Read the generated public key const publicKeyPath = keyPath + '.pub'; const publicKey = readFileSync(publicKeyPath, 'utf8'); - + // Set proper permissions chmodSync(keyPath, 0o600); chmodSync(publicKeyPath, 0o644); - + resolve({ privateKey, publicKey: publicKey.trim() @@ -725,11 +725,11 @@ expect { */ getPublicKey(keyPath) { const publicKeyPath = keyPath + '.pub'; - + if (!existsSync(publicKeyPath)) { throw new Error('Public key file not found'); } - + return readFileSync(publicKeyPath, 'utf8').trim(); }