From 971da5988ac28fcd0be6d11df267e49529151830 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 23 Mar 2023 20:20:24 -0700 Subject: [PATCH] Standardize syntax, name casing and fix package versioning feature. --- install_and_cache_pkgs.sh | 32 +++++++------ lib.sh | 96 +++++++++++++++++++-------------------- post_cache_action.sh | 4 +- restore_pkgs.sh | 16 +++---- 4 files changed, 76 insertions(+), 72 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 3a6eb0d..609e374 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -6,7 +6,7 @@ set -e # Debug mode for diagnosing issues. # Setup first before other operations. debug="${2}" -test ${debug} == "true" && set -x +test "${debug}" = "true" && set -x # Include library. script_dir="$(dirname -- "$(realpath -- "${0}")")" @@ -18,17 +18,20 @@ cache_dir="${1}" # List of the packages to use. input_packages="${@:3}" -# Trim commas, excess spaces, and sort. -normalized_packages="$(normalize_package_list "${input_packages}")" +# Trim commas, excess spaces, sort, and version syntax. +# +# NOTE: Unless specified, all APT package listings of name and version use +# colon delimited and not equals delimited syntax (i.e. [:=]). +packages="$(get_normalized_package_list "${input_packages}")" -package_count=$(wc -w <<< "${normalized_packages}") +package_count=$(wc -w <<< "${packages}") log "Clean installing and caching ${package_count} package(s)." log_empty_line manifest_main="" log "Package list:" -for package in ${normalized_packages}; do +for package in ${packages}; do read package_name package_ver < <(get_package_name_ver "${package}") manifest_main="${manifest_main}${package_name}:${package_ver}," log "- ${package_name}:${package_ver}" @@ -63,8 +66,9 @@ manifest_all="" install_log_filepath="${cache_dir}/install.log" log "Clean installing ${package_count} packages..." +apt_syntax_packages="$(convert_action_to_apt_syntax_packages "${packages}")" # Zero interaction while installing or upgrading the system via apt. -sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${normalized_packages} > "${install_log_filepath}" +sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${apt_syntax_packages} > "${install_log_filepath}" log "done" log "Installation log written to ${install_log_filepath}" @@ -78,20 +82,20 @@ done log_empty_line -installed_package_count=$(wc -w <<< "${installed_packages}") -log "Caching ${installed_package_count} installed packages..." +installed_packages_count=$(wc -w <<< "${installed_packages}") +log "Caching ${installed_packages_count} installed packages..." for installed_package in ${installed_packages}; do cache_filepath="${cache_dir}/${installed_package}.tar" # Sanity test in case APT enumerates duplicates. if test ! -f "${cache_filepath}"; then - read installed_package_name installed_package_ver < <(get_package_name_ver "${installed_package}") - log " * Caching ${installed_package_name} to ${cache_filepath}..." + read package_name package_ver < <(get_package_name_ver "${installed_package}") + log " * Caching ${package_name} to ${cache_filepath}..." # Pipe all package files (no folders) and installation control data to Tar. - { dpkg -L "${installed_package_name}" \ - & get_install_filepath "" "${package_name}" "preinst" \ - & get_install_filepath "" "${package_name}" "postinst"; } | + { dpkg -L "${package_name}" \ + & get_install_script_filepath "" "${package_name}" "preinst" \ + & get_install_script_filepath "" "${package_name}" "postinst"; } | while IFS= read -r f; do test -f "${f}" -o -L "${f}" && get_tar_relpath "${f}"; done | xargs -I {} echo \'{}\' | # Single quotes ensure literals like backslash get captured. sudo xargs tar -cf "${cache_filepath}" -C / @@ -100,7 +104,7 @@ for installed_package in ${installed_packages}; do fi # Comma delimited name:ver pairs in the all packages manifest. - manifest_all="${manifest_all}${installed_package_name}:${installed_package_ver}," + manifest_all="${manifest_all}${package_name}:${package_ver}," done log "done (total cache size $(du -h ${cache_dir} | tail -1 | awk '{print $1}'))" diff --git a/lib.sh b/lib.sh index 6f85fe7..b4e8040 100755 --- a/lib.sh +++ b/lib.sh @@ -1,5 +1,17 @@ #!/bin/bash +############################################################################### +# Convert the APT syntax package (= delimited) to action syntax package +# (: delimited). +# Arguments: +# APT syntax package, with or without version. +# Returns: +# Action syntax package, with or without version. +############################################################################### +function convert_action_to_apt_syntax_packages() { + echo ${1} | sed 's/:/=/g' +} + ############################################################################### # Execute the Debian install script. # Arguments: @@ -13,7 +25,7 @@ function execute_install_script { local package_name=$(basename ${2} | awk -F\: '{print $1}') local install_script_filepath=$(\ - get_install_filepath "${1}" "${package_name}" "${3}") + get_install_script_filepath "${1}" "${package_name}" "${3}") if test ! -z "${install_script_filepath}"; then log "- Executing ${install_script_filepath}..." # Don't abort on errors; dpkg-trigger will error normally since it is @@ -23,19 +35,37 @@ function execute_install_script { fi } +############################################################################### +# Gets the Debian install script filepath. +# Arguments: +# Root directory to search from. +# Name of the unqualified package to search for. +# Extension of the installation script (preinst, postinst) +# Returns: +# Filepath of the script file, otherwise an empty string. +############################################################################### +function get_install_script_filepath { + # Filename includes arch (e.g. amd64). + local filepath="$(\ + ls -1 ${1}var/lib/dpkg/info/${2}*.${3} 2> /dev/null \ + | grep -E ${2}'(:.*)?.'${3} | head -1 || true)" + test "${filepath}" && echo "${filepath}" +} + ############################################################################### # Gets a list of installed packages from a Debian package installation log. # Arguments: # The filepath of the Debian install log. # Returns: -# The list of space delimited pairs with each pair colon delimited. -# : ... +# The list of colon delimited action syntax pairs with each pair equals +# delimited. : :... ############################################################################### function get_installed_packages { local install_log_filepath="${1}" local regex="^Unpacking ([^ :]+)([^ ]+)? (\[[^ ]+\]\s)?\(([^ )]+)" local dep_packages="" while read -r line; do + # ${regex} should be unquoted since it isn't a literal. if [[ "${line}" =~ ${regex} ]]; then dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[4]} " else @@ -51,48 +81,38 @@ function get_installed_packages { } ############################################################################### -# Splits a fully qualified package into name and version. +# Splits a fully action syntax APT package into the name and version. # Arguments: -# The colon delimited package pair or just the package name. +# The action syntax colon delimited package pair or just the package name. # Returns: # The package name and version pair. ############################################################################### function get_package_name_ver { + local ORIG_IFS="${IFS}" IFS=\: read name ver <<< "${1}" # If version not found in the fully qualified package value. if test -z "${ver}"; then ver="$(grep "Version:" <<< "$(apt-cache show ${name})" | awk '{print $2}')" fi echo "${name}" "${ver}" + IFS="${ORIG_IFS}" } ############################################################################### -# Gets the package name from the cached package filepath in the -# path/to/cache/dir/:.tar format. +# Sorts given packages by name and split on commas and/or spaces. # Arguments: -# Filepath to the cached packaged. +# The comma and/or space delimited list of packages. # Returns: -# The package name. +# Sorted list of space delimited packages. ############################################################################### -function get_package_name_from_cached_filepath { - basename ${cached_pkg_filepath} | awk -F\: '{print $1}' -} - -############################################################################### -# Gets the Debian install script file location. -# Arguments: -# Root directory to search from. -# Name of the unqualified package to search for. -# Extension of the installation script (preinst, postinst) -# Returns: -# Filepath of the script file, otherwise an empty string. -############################################################################### -function get_install_filepath { - # Filename includes arch (e.g. amd64). - local filepath="$(\ - ls -1 ${1}var/lib/dpkg/info/${2}*.${3} 2> /dev/null \ - | grep -E ${2}'(:.*)?.'${3} | head -1 || true)" - test "${filepath}" && echo "${filepath}" +function get_normalized_package_list { + # Remove commas, and block scalar folded backslashes. + local stripped=$(echo "${1}" | sed 's/[,\]/ /g') + # Remove extraneous spaces at the middle, beginning, and end. + local trimmed="$(\ + echo "${stripped}" \ + | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" + echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ' } ############################################################################### @@ -117,26 +137,6 @@ function log_err { >&2 echo "$(date +%H:%M:%S)" "${@}"; } function log_empty_line { echo ""; } -############################################################################### -# Sorts given packages by name and split on commas and/or spaces. -# Arguments: -# The comma and/or space delimited list of packages. -# Returns: -# Sorted list of space delimited packages. -############################################################################### -function normalize_package_list { - # Remove commas, and block scalar folded backslashes. - local stripped=$(echo "${1}" | sed 's/[,\]/ /g') - # Remove extraneous spaces at the middle, beginning, and end. - local trimmed="$(\ - echo "${stripped}" \ - | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" - local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')" - # Convert colon delimited package version pairs to APT syntax (i.e. =). - local versioned="$(echo $sorted | sed 's/:/=/g')" - echo "${sorted}" -} - ############################################################################### # Validates an argument to be of a boolean value. # Arguments: diff --git a/post_cache_action.sh b/post_cache_action.sh index 25e7da3..f71c336 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -23,12 +23,12 @@ execute_install_scripts="${4}" # Debug mode for diagnosing issues. debug="${5}" -test ${debug} == "true" && set -x +test "${debug}" = "true" && set -x # List of the packages to use. packages="${@:6}" -if [ "$cache_hit" == true ]; then +if test "${cache_hit}" = "true"; then ${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}" else ${script_dir}/install_and_cache_pkgs.sh "${cache_dir}" "${debug}" ${packages} diff --git a/restore_pkgs.sh b/restore_pkgs.sh index fec5b29..418f9f0 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -40,22 +40,22 @@ log "done" log_empty_line # Only search for archived results. Manifest and cache key also live here. -cached_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar | sort) -cached_pkg_filecount=$(echo ${cached_pkg_filepaths} | wc -w) +cached_filepaths=$(ls -1 "${cache_dir}"/*.tar | sort) +cached_filecount=$(echo ${cached_filepaths} | wc -w) -log "Restoring ${cached_pkg_filecount} packages from cache..." -for cached_pkg_filepath in ${cached_pkg_filepaths}; do +log "Restoring ${cached_filecount} packages from cache..." +for cached_filepath in ${cached_filepaths}; do - log "- $(basename "${cached_pkg_filepath}") restoring..." - sudo tar -xf "${cached_pkg_filepath}" -C "${cache_restore_root}" > /dev/null + log "- $(basename "${cached_filepath}") restoring..." + sudo tar -xf "${cached_filepath}" -C "${cache_restore_root}" > /dev/null log " done" # Execute install scripts if available. if test ${execute_install_scripts} == "true"; then # May have to add more handling for extracting pre-install script before extracting all files. # Keeping it simple for now. - execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" preinst install - execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" postinst configure + execute_install_script "${cache_restore_root}" "${cached_filepath}" preinst install + execute_install_script "${cache_restore_root}" "${cached_filepath}" postinst configure fi done log "done"