diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index d85381d..bf12b74 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -6,7 +6,7 @@ set -e # Debug mode for diagnosing issues. # Setup first before other operations. debug="${2}" -test ${debug} == "true" && set -x +test "${debug}" = "true" && set -x # Include library. script_dir="$(dirname -- "$(realpath -- "${0}")")" @@ -18,20 +18,23 @@ cache_dir="${1}" # List of the packages to use. input_packages="${@:3}" -# Trim commas, excess spaces, and sort. -normalized_packages="$(normalize_package_list "${input_packages}")" +# Trim commas, excess spaces, sort, and version syntax. +# +# NOTE: Unless specified, all APT package listings of name and version use +# colon delimited and not equals delimited syntax (i.e. [:=]). +packages="$(get_normalized_package_list "${input_packages}")" -package_count=$(wc -w <<< "${normalized_packages}") +package_count=$(wc -w <<< "${packages}") log "Clean installing and caching ${package_count} package(s)." log_empty_line manifest_main="" log "Package list:" -for package in ${normalized_packages}; do +for package in ${packages}; do read package_name package_ver < <(get_package_name_ver "${package}") - manifest_main="${manifest_main}${package_name}:${package_ver}," - log "- ${package_name}:${package_ver}" + manifest_main="${manifest_main}${package_name}=${package_ver}," + log "- ${package_name} (${package_ver})" done write_manifest "main" "${manifest_main}" "${cache_dir}/manifest_main.log" @@ -65,7 +68,7 @@ install_log_filepath="${cache_dir}/install.log" log "Clean installing ${package_count} packages..." # Zero interaction while installing or upgrading the system via apt. -sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${normalized_packages} > "${install_log_filepath}" +sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${packages} > "${install_log_filepath}" log "done" log "Installation log written to ${install_log_filepath}" @@ -74,34 +77,35 @@ log_empty_line installed_packages=$(get_installed_packages "${install_log_filepath}") log "Installed package list:" for installed_package in ${installed_packages}; do - log "- ${installed_package}" + # Reformat for human friendly reading. + log "- $(echo ${installed_package} | awk -F\= '{print $1" ("$2")"}')" done log_empty_line -installed_package_count=$(wc -w <<< "${installed_packages}") -log "Caching ${installed_package_count} installed packages..." +installed_packages_count=$(wc -w <<< "${installed_packages}") +log "Caching ${installed_packages_count} installed packages..." for installed_package in ${installed_packages}; do cache_filepath="${cache_dir}/${installed_package}.tar" # Sanity test in case APT enumerates duplicates. if test ! -f "${cache_filepath}"; then - read installed_package_name installed_package_ver < <(get_package_name_ver "${installed_package}") - log " * Caching ${installed_package_name} to ${cache_filepath}..." + read package_name package_ver < <(get_package_name_ver "${installed_package}") + log " * Caching ${package_name} to ${cache_filepath}..." # Pipe all package files (no folders) and installation control data to Tar. - { dpkg -L "${installed_package_name}" \ - & get_install_filepath "" "${package_name}" "preinst" \ - & get_install_filepath "" "${package_name}" "postinst"; } | + { dpkg -L "${package_name}" \ + & get_install_script_filepath "" "${package_name}" "preinst" \ + & get_install_script_filepath "" "${package_name}" "postinst"; } | while IFS= read -r f; do test -f "${f}" -o -L "${f}" && get_tar_relpath "${f}"; done | - xargs -I {} echo \"{}\" | + xargs -I {} echo \'{}\' | # Single quotes ensure literals like backslash get captured. sudo xargs tar -cf "${cache_filepath}" -C / log " done (compressed size $(du -h "${cache_filepath}" | cut -f1))." fi # Comma delimited name:ver pairs in the all packages manifest. - manifest_all="${manifest_all}${installed_package_name}:${installed_package_ver}," + manifest_all="${manifest_all}${package_name}=${package_ver}," done log "done (total cache size $(du -h ${cache_dir} | tail -1 | awk '{print $1}'))" diff --git a/lib.sh b/lib.sh index aded419..2144320 100755 --- a/lib.sh +++ b/lib.sh @@ -11,9 +11,9 @@ # Filepath of the install script, otherwise an empty string. ############################################################################### function execute_install_script { - local package_name=$(basename ${2} | awk -F\: '{print $1}') + local package_name=$(basename ${2} | awk -F\= '{print $1}') local install_script_filepath=$(\ - get_install_filepath "${1}" "${package_name}" "${3}") + get_install_script_filepath "${1}" "${package_name}" "${3}") if test ! -z "${install_script_filepath}"; then log "- Executing ${install_script_filepath}..." # Don't abort on errors; dpkg-trigger will error normally since it is @@ -23,21 +23,39 @@ function execute_install_script { fi } +############################################################################### +# Gets the Debian install script filepath. +# Arguments: +# Root directory to search from. +# Name of the unqualified package to search for. +# Extension of the installation script (preinst, postinst) +# Returns: +# Filepath of the script file, otherwise an empty string. +############################################################################### +function get_install_script_filepath { + # Filename includes arch (e.g. amd64). + local filepath="$(\ + ls -1 ${1}var/lib/dpkg/info/${2}*.${3} 2> /dev/null \ + | grep -E ${2}'(:.*)?.'${3} | head -1 || true)" + test "${filepath}" && echo "${filepath}" +} + ############################################################################### # Gets a list of installed packages from a Debian package installation log. # Arguments: # The filepath of the Debian install log. # Returns: -# The list of space delimited pairs with each pair colon delimited. -# : ... +# The list of colon delimited action syntax pairs with each pair equals +# delimited. : :... ############################################################################### function get_installed_packages { local install_log_filepath="${1}" local regex="^Unpacking ([^ :]+)([^ ]+)? (\[[^ ]+\]\s)?\(([^ )]+)" local dep_packages="" while read -r line; do + # ${regex} should be unquoted since it isn't a literal. if [[ "${line}" =~ ${regex} ]]; then - dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[4]} " + dep_packages="${dep_packages}${BASH_REMATCH[1]}=${BASH_REMATCH[4]} " else log_err "Unable to parse package name and version from \"${line}\"" exit 2 @@ -51,48 +69,38 @@ function get_installed_packages { } ############################################################################### -# Splits a fully qualified package into name and version. +# Splits a fully action syntax APT package into the name and version. # Arguments: -# The colon delimited package pair or just the package name. +# The action syntax colon delimited package pair or just the package name. # Returns: # The package name and version pair. ############################################################################### function get_package_name_ver { - IFS=\: read name ver <<< "${1}" + local ORIG_IFS="${IFS}" + IFS=\= read name ver <<< "${1}" + IFS="${ORIG_IFS}" # If version not found in the fully qualified package value. if test -z "${ver}"; then ver="$(grep "Version:" <<< "$(apt-cache show ${name})" | awk '{print $2}')" fi - echo "${name}" "${ver}" + echo "${name}" "${ver}" } ############################################################################### -# Gets the package name from the cached package filepath in the -# path/to/cache/dir/:.tar format. +# Sorts given packages by name and split on commas and/or spaces. # Arguments: -# Filepath to the cached packaged. +# The comma and/or space delimited list of packages. # Returns: -# The package name. +# Sorted list of space delimited packages. ############################################################################### -function get_package_name_from_cached_filepath { - basename ${cached_pkg_filepath} | awk -F\: '{print $1}' -} - -############################################################################### -# Gets the Debian install script file location. -# Arguments: -# Root directory to search from. -# Name of the unqualified package to search for. -# Extension of the installation script (preinst, postinst) -# Returns: -# Filepath of the script file, otherwise an empty string. -############################################################################### -function get_install_filepath { - # Filename includes arch (e.g. amd64). - local filepath="$(\ - ls -1 ${1}var/lib/dpkg/info/${2}*.${3} 2> /dev/null \ - | grep -E ${2}'(:.*)?.'${3} | head -1 || true)" - test "${filepath}" && echo "${filepath}" +function get_normalized_package_list { + # Remove commas, and block scalar folded backslashes. + local stripped=$(echo "${1}" | sed 's/[,\]/ /g') + # Remove extraneous spaces at the middle, beginning, and end. + local trimmed="$(\ + echo "${stripped}" \ + | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" + echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ' } ############################################################################### @@ -117,24 +125,6 @@ function log_err { >&2 echo "$(date +%H:%M:%S)" "${@}"; } function log_empty_line { echo ""; } -############################################################################### -# Sorts given packages by name and split on commas and/or spaces. -# Arguments: -# The comma and/or space delimited list of packages. -# Returns: -# Sorted list of space delimited packages. -############################################################################### -function normalize_package_list { - # Remove commas, and block scalar folded backslashes. - local stripped=$(echo "${1}" | sed 's/[,\]/ /g') - # Remove extraneous spaces at the middle, beginning, and end. - local trimmed="$(\ - echo "${stripped}" \ - | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" - local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')" - echo "${sorted}" -} - ############################################################################### # Validates an argument to be of a boolean value. # Arguments: diff --git a/post_cache_action.sh b/post_cache_action.sh index 25e7da3..f71c336 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -23,12 +23,12 @@ execute_install_scripts="${4}" # Debug mode for diagnosing issues. debug="${5}" -test ${debug} == "true" && set -x +test "${debug}" = "true" && set -x # List of the packages to use. packages="${@:6}" -if [ "$cache_hit" == true ]; then +if test "${cache_hit}" = "true"; then ${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}" else ${script_dir}/install_and_cache_pkgs.sh "${cache_dir}" "${debug}" ${packages} diff --git a/pre_cache_action.sh b/pre_cache_action.sh index bb86929..b487743 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -28,7 +28,7 @@ debug="${4}" input_packages="${@:5}" # Trim commas, excess spaces, and sort. -packages="$(normalize_package_list "${input_packages}")" +packages="$(get_normalized_package_list "${input_packages}")" # Create cache directory so artifacts can be saved. mkdir -p ${cache_dir} @@ -55,8 +55,8 @@ log_empty_line versioned_packages="" log "Verifying packages..." -for package in ${packages}; do - if test ! "$(apt-cache show "${package}")"; then +for package in ${packages}; do + if test ! "$(apt-cache show ${package})"; then echo "aborted" log "Package '${package}' not found." >&2 exit 5 @@ -74,12 +74,12 @@ set -e log "Creating cache key..." # TODO Can we prove this will happen again? -normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")" +normalized_versioned_packages="$(get_normalized_package_list "${versioned_packages}")" log "- Normalized package list is '${normalized_versioned_packages}'." # Forces an update in cases where an accidental breaking change was introduced # and a global cache reset is required. -force_update_inc="0" +force_update_inc="1" value="${normalized_versioned_packages} @ ${version} ${force_update_inc}" log "- Value to hash is '${value}'." diff --git a/restore_pkgs.sh b/restore_pkgs.sh index fec5b29..418f9f0 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -40,22 +40,22 @@ log "done" log_empty_line # Only search for archived results. Manifest and cache key also live here. -cached_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar | sort) -cached_pkg_filecount=$(echo ${cached_pkg_filepaths} | wc -w) +cached_filepaths=$(ls -1 "${cache_dir}"/*.tar | sort) +cached_filecount=$(echo ${cached_filepaths} | wc -w) -log "Restoring ${cached_pkg_filecount} packages from cache..." -for cached_pkg_filepath in ${cached_pkg_filepaths}; do +log "Restoring ${cached_filecount} packages from cache..." +for cached_filepath in ${cached_filepaths}; do - log "- $(basename "${cached_pkg_filepath}") restoring..." - sudo tar -xf "${cached_pkg_filepath}" -C "${cache_restore_root}" > /dev/null + log "- $(basename "${cached_filepath}") restoring..." + sudo tar -xf "${cached_filepath}" -C "${cache_restore_root}" > /dev/null log " done" # Execute install scripts if available. if test ${execute_install_scripts} == "true"; then # May have to add more handling for extracting pre-install script before extracting all files. # Keeping it simple for now. - execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" preinst install - execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" postinst configure + execute_install_script "${cache_restore_root}" "${cached_filepath}" preinst install + execute_install_script "${cache_restore_root}" "${cached_filepath}" postinst configure fi done log "done"