From a644619d1f0e6b19aadae0ce5ef7fea7cacdf9f3 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Mon, 13 Mar 2023 21:24:13 -0700 Subject: [PATCH 1/7] Enclose filenames in single quotes to capture literals #99 --- install_and_cache_pkgs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index c430222..3a6eb0d 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -93,7 +93,7 @@ for installed_package in ${installed_packages}; do & get_install_filepath "" "${package_name}" "preinst" \ & get_install_filepath "" "${package_name}" "postinst"; } | while IFS= read -r f; do test -f "${f}" -o -L "${f}" && get_tar_relpath "${f}"; done | - xargs -I {} echo \"{}\" | + xargs -I {} echo \'{}\' | # Single quotes ensure literals like backslash get captured. sudo xargs tar -cf "${cache_filepath}" -C / log " done (compressed size $(du -h "${cache_filepath}" | cut -f1))." From bd5455834e813eff43a50c60e09b7dd661a0db80 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Mon, 13 Mar 2023 21:43:46 -0700 Subject: [PATCH 2/7] Convert action to APT version syntax. --- lib.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib.sh b/lib.sh index aded419..6f85fe7 100755 --- a/lib.sh +++ b/lib.sh @@ -132,6 +132,8 @@ function normalize_package_list { echo "${stripped}" \ | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')" + # Convert colon delimited package version pairs to APT syntax (i.e. =). + local versioned="$(echo $sorted | sed 's/:/=/g')" echo "${sorted}" } From 971da5988ac28fcd0be6d11df267e49529151830 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 23 Mar 2023 20:20:24 -0700 Subject: [PATCH 3/7] Standardize syntax, name casing and fix package versioning feature. --- install_and_cache_pkgs.sh | 32 +++++++------ lib.sh | 96 +++++++++++++++++++-------------------- post_cache_action.sh | 4 +- restore_pkgs.sh | 16 +++---- 4 files changed, 76 insertions(+), 72 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 3a6eb0d..609e374 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -6,7 +6,7 @@ set -e # Debug mode for diagnosing issues. # Setup first before other operations. debug="${2}" -test ${debug} == "true" && set -x +test "${debug}" = "true" && set -x # Include library. script_dir="$(dirname -- "$(realpath -- "${0}")")" @@ -18,17 +18,20 @@ cache_dir="${1}" # List of the packages to use. input_packages="${@:3}" -# Trim commas, excess spaces, and sort. -normalized_packages="$(normalize_package_list "${input_packages}")" +# Trim commas, excess spaces, sort, and version syntax. +# +# NOTE: Unless specified, all APT package listings of name and version use +# colon delimited and not equals delimited syntax (i.e. [:=]). +packages="$(get_normalized_package_list "${input_packages}")" -package_count=$(wc -w <<< "${normalized_packages}") +package_count=$(wc -w <<< "${packages}") log "Clean installing and caching ${package_count} package(s)." log_empty_line manifest_main="" log "Package list:" -for package in ${normalized_packages}; do +for package in ${packages}; do read package_name package_ver < <(get_package_name_ver "${package}") manifest_main="${manifest_main}${package_name}:${package_ver}," log "- ${package_name}:${package_ver}" @@ -63,8 +66,9 @@ manifest_all="" install_log_filepath="${cache_dir}/install.log" log "Clean installing ${package_count} packages..." +apt_syntax_packages="$(convert_action_to_apt_syntax_packages "${packages}")" # Zero interaction while installing or upgrading the system via apt. -sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${normalized_packages} > "${install_log_filepath}" +sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${apt_syntax_packages} > "${install_log_filepath}" log "done" log "Installation log written to ${install_log_filepath}" @@ -78,20 +82,20 @@ done log_empty_line -installed_package_count=$(wc -w <<< "${installed_packages}") -log "Caching ${installed_package_count} installed packages..." +installed_packages_count=$(wc -w <<< "${installed_packages}") +log "Caching ${installed_packages_count} installed packages..." for installed_package in ${installed_packages}; do cache_filepath="${cache_dir}/${installed_package}.tar" # Sanity test in case APT enumerates duplicates. if test ! -f "${cache_filepath}"; then - read installed_package_name installed_package_ver < <(get_package_name_ver "${installed_package}") - log " * Caching ${installed_package_name} to ${cache_filepath}..." + read package_name package_ver < <(get_package_name_ver "${installed_package}") + log " * Caching ${package_name} to ${cache_filepath}..." # Pipe all package files (no folders) and installation control data to Tar. - { dpkg -L "${installed_package_name}" \ - & get_install_filepath "" "${package_name}" "preinst" \ - & get_install_filepath "" "${package_name}" "postinst"; } | + { dpkg -L "${package_name}" \ + & get_install_script_filepath "" "${package_name}" "preinst" \ + & get_install_script_filepath "" "${package_name}" "postinst"; } | while IFS= read -r f; do test -f "${f}" -o -L "${f}" && get_tar_relpath "${f}"; done | xargs -I {} echo \'{}\' | # Single quotes ensure literals like backslash get captured. sudo xargs tar -cf "${cache_filepath}" -C / @@ -100,7 +104,7 @@ for installed_package in ${installed_packages}; do fi # Comma delimited name:ver pairs in the all packages manifest. - manifest_all="${manifest_all}${installed_package_name}:${installed_package_ver}," + manifest_all="${manifest_all}${package_name}:${package_ver}," done log "done (total cache size $(du -h ${cache_dir} | tail -1 | awk '{print $1}'))" diff --git a/lib.sh b/lib.sh index 6f85fe7..b4e8040 100755 --- a/lib.sh +++ b/lib.sh @@ -1,5 +1,17 @@ #!/bin/bash +############################################################################### +# Convert the APT syntax package (= delimited) to action syntax package +# (: delimited). +# Arguments: +# APT syntax package, with or without version. +# Returns: +# Action syntax package, with or without version. +############################################################################### +function convert_action_to_apt_syntax_packages() { + echo ${1} | sed 's/:/=/g' +} + ############################################################################### # Execute the Debian install script. # Arguments: @@ -13,7 +25,7 @@ function execute_install_script { local package_name=$(basename ${2} | awk -F\: '{print $1}') local install_script_filepath=$(\ - get_install_filepath "${1}" "${package_name}" "${3}") + get_install_script_filepath "${1}" "${package_name}" "${3}") if test ! -z "${install_script_filepath}"; then log "- Executing ${install_script_filepath}..." # Don't abort on errors; dpkg-trigger will error normally since it is @@ -23,19 +35,37 @@ function execute_install_script { fi } +############################################################################### +# Gets the Debian install script filepath. +# Arguments: +# Root directory to search from. +# Name of the unqualified package to search for. +# Extension of the installation script (preinst, postinst) +# Returns: +# Filepath of the script file, otherwise an empty string. +############################################################################### +function get_install_script_filepath { + # Filename includes arch (e.g. amd64). + local filepath="$(\ + ls -1 ${1}var/lib/dpkg/info/${2}*.${3} 2> /dev/null \ + | grep -E ${2}'(:.*)?.'${3} | head -1 || true)" + test "${filepath}" && echo "${filepath}" +} + ############################################################################### # Gets a list of installed packages from a Debian package installation log. # Arguments: # The filepath of the Debian install log. # Returns: -# The list of space delimited pairs with each pair colon delimited. -# : ... +# The list of colon delimited action syntax pairs with each pair equals +# delimited. : :... ############################################################################### function get_installed_packages { local install_log_filepath="${1}" local regex="^Unpacking ([^ :]+)([^ ]+)? (\[[^ ]+\]\s)?\(([^ )]+)" local dep_packages="" while read -r line; do + # ${regex} should be unquoted since it isn't a literal. if [[ "${line}" =~ ${regex} ]]; then dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[4]} " else @@ -51,48 +81,38 @@ function get_installed_packages { } ############################################################################### -# Splits a fully qualified package into name and version. +# Splits a fully action syntax APT package into the name and version. # Arguments: -# The colon delimited package pair or just the package name. +# The action syntax colon delimited package pair or just the package name. # Returns: # The package name and version pair. ############################################################################### function get_package_name_ver { + local ORIG_IFS="${IFS}" IFS=\: read name ver <<< "${1}" # If version not found in the fully qualified package value. if test -z "${ver}"; then ver="$(grep "Version:" <<< "$(apt-cache show ${name})" | awk '{print $2}')" fi echo "${name}" "${ver}" + IFS="${ORIG_IFS}" } ############################################################################### -# Gets the package name from the cached package filepath in the -# path/to/cache/dir/:.tar format. +# Sorts given packages by name and split on commas and/or spaces. # Arguments: -# Filepath to the cached packaged. +# The comma and/or space delimited list of packages. # Returns: -# The package name. +# Sorted list of space delimited packages. ############################################################################### -function get_package_name_from_cached_filepath { - basename ${cached_pkg_filepath} | awk -F\: '{print $1}' -} - -############################################################################### -# Gets the Debian install script file location. -# Arguments: -# Root directory to search from. -# Name of the unqualified package to search for. -# Extension of the installation script (preinst, postinst) -# Returns: -# Filepath of the script file, otherwise an empty string. -############################################################################### -function get_install_filepath { - # Filename includes arch (e.g. amd64). - local filepath="$(\ - ls -1 ${1}var/lib/dpkg/info/${2}*.${3} 2> /dev/null \ - | grep -E ${2}'(:.*)?.'${3} | head -1 || true)" - test "${filepath}" && echo "${filepath}" +function get_normalized_package_list { + # Remove commas, and block scalar folded backslashes. + local stripped=$(echo "${1}" | sed 's/[,\]/ /g') + # Remove extraneous spaces at the middle, beginning, and end. + local trimmed="$(\ + echo "${stripped}" \ + | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" + echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ' } ############################################################################### @@ -117,26 +137,6 @@ function log_err { >&2 echo "$(date +%H:%M:%S)" "${@}"; } function log_empty_line { echo ""; } -############################################################################### -# Sorts given packages by name and split on commas and/or spaces. -# Arguments: -# The comma and/or space delimited list of packages. -# Returns: -# Sorted list of space delimited packages. -############################################################################### -function normalize_package_list { - # Remove commas, and block scalar folded backslashes. - local stripped=$(echo "${1}" | sed 's/[,\]/ /g') - # Remove extraneous spaces at the middle, beginning, and end. - local trimmed="$(\ - echo "${stripped}" \ - | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" - local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')" - # Convert colon delimited package version pairs to APT syntax (i.e. =). - local versioned="$(echo $sorted | sed 's/:/=/g')" - echo "${sorted}" -} - ############################################################################### # Validates an argument to be of a boolean value. # Arguments: diff --git a/post_cache_action.sh b/post_cache_action.sh index 25e7da3..f71c336 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -23,12 +23,12 @@ execute_install_scripts="${4}" # Debug mode for diagnosing issues. debug="${5}" -test ${debug} == "true" && set -x +test "${debug}" = "true" && set -x # List of the packages to use. packages="${@:6}" -if [ "$cache_hit" == true ]; then +if test "${cache_hit}" = "true"; then ${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}" else ${script_dir}/install_and_cache_pkgs.sh "${cache_dir}" "${debug}" ${packages} diff --git a/restore_pkgs.sh b/restore_pkgs.sh index fec5b29..418f9f0 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -40,22 +40,22 @@ log "done" log_empty_line # Only search for archived results. Manifest and cache key also live here. -cached_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar | sort) -cached_pkg_filecount=$(echo ${cached_pkg_filepaths} | wc -w) +cached_filepaths=$(ls -1 "${cache_dir}"/*.tar | sort) +cached_filecount=$(echo ${cached_filepaths} | wc -w) -log "Restoring ${cached_pkg_filecount} packages from cache..." -for cached_pkg_filepath in ${cached_pkg_filepaths}; do +log "Restoring ${cached_filecount} packages from cache..." +for cached_filepath in ${cached_filepaths}; do - log "- $(basename "${cached_pkg_filepath}") restoring..." - sudo tar -xf "${cached_pkg_filepath}" -C "${cache_restore_root}" > /dev/null + log "- $(basename "${cached_filepath}") restoring..." + sudo tar -xf "${cached_filepath}" -C "${cache_restore_root}" > /dev/null log " done" # Execute install scripts if available. if test ${execute_install_scripts} == "true"; then # May have to add more handling for extracting pre-install script before extracting all files. # Keeping it simple for now. - execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" preinst install - execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" postinst configure + execute_install_script "${cache_restore_root}" "${cached_filepath}" preinst install + execute_install_script "${cache_restore_root}" "${cached_filepath}" postinst configure fi done log "done" From cdad9718507a9a547abefb303f86c005642cac49 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 23 Mar 2023 20:50:29 -0700 Subject: [PATCH 4/7] Fix broken function calls. --- pre_cache_action.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pre_cache_action.sh b/pre_cache_action.sh index bb86929..328825f 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -28,7 +28,7 @@ debug="${4}" input_packages="${@:5}" # Trim commas, excess spaces, and sort. -packages="$(normalize_package_list "${input_packages}")" +packages="$(get_normalized_package_list "${input_packages}")" # Create cache directory so artifacts can be saved. mkdir -p ${cache_dir} @@ -55,8 +55,9 @@ log_empty_line versioned_packages="" log "Verifying packages..." -for package in ${packages}; do - if test ! "$(apt-cache show "${package}")"; then +for package in ${packages}; do + apt_syntax_package=$(convert_action_to_apt_syntax_packages ${package}) + if test ! "$(apt-cache show ${apt_syntax_package})"; then echo "aborted" log "Package '${package}' not found." >&2 exit 5 @@ -74,7 +75,7 @@ set -e log "Creating cache key..." # TODO Can we prove this will happen again? -normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")" +normalized_versioned_packages="$(get_normalized_package_list "${versioned_packages}")" log "- Normalized package list is '${normalized_versioned_packages}'." # Forces an update in cases where an accidental breaking change was introduced From f5bcdd76d30d23a76e68682f0d12bbcd1442450a Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 23 Mar 2023 22:19:43 -0700 Subject: [PATCH 5/7] Use APT syntax for name version delimitation and not a colon. --- install_and_cache_pkgs.sh | 14 +++++++------- lib.sh | 22 +++++----------------- pre_cache_action.sh | 5 ++--- 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 609e374..0ec55d6 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -33,8 +33,8 @@ manifest_main="" log "Package list:" for package in ${packages}; do read package_name package_ver < <(get_package_name_ver "${package}") - manifest_main="${manifest_main}${package_name}:${package_ver}," - log "- ${package_name}:${package_ver}" + manifest_main="${manifest_main}${package_name}=${package_ver}," + log "- ${package_name} (${package_ver})" done write_manifest "main" "${manifest_main}" "${cache_dir}/manifest_main.log" @@ -42,7 +42,7 @@ log_empty_line log "Installing apt-fast for optimized installs..." # Install apt-fast for optimized installs. -/bin/bash -c "$(curl -sL https://git.io/vokNn)" +# /bin/bash -c "$(curl -sL https://git.io/vokNn)" log "done" log_empty_line @@ -66,9 +66,8 @@ manifest_all="" install_log_filepath="${cache_dir}/install.log" log "Clean installing ${package_count} packages..." -apt_syntax_packages="$(convert_action_to_apt_syntax_packages "${packages}")" # Zero interaction while installing or upgrading the system via apt. -sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${apt_syntax_packages} > "${install_log_filepath}" +sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${packages} > "${install_log_filepath}" log "done" log "Installation log written to ${install_log_filepath}" @@ -77,7 +76,8 @@ log_empty_line installed_packages=$(get_installed_packages "${install_log_filepath}") log "Installed package list:" for installed_package in ${installed_packages}; do - log "- ${installed_package}" + # Reformat for human friendly reading. + log "- $(echo ${installed_package} | awk -F\= '{print $1" ("$2")"}')" done log_empty_line @@ -104,7 +104,7 @@ for installed_package in ${installed_packages}; do fi # Comma delimited name:ver pairs in the all packages manifest. - manifest_all="${manifest_all}${package_name}:${package_ver}," + manifest_all="${manifest_all}${package_name}=${package_ver}," done log "done (total cache size $(du -h ${cache_dir} | tail -1 | awk '{print $1}'))" diff --git a/lib.sh b/lib.sh index b4e8040..2144320 100755 --- a/lib.sh +++ b/lib.sh @@ -1,17 +1,5 @@ #!/bin/bash -############################################################################### -# Convert the APT syntax package (= delimited) to action syntax package -# (: delimited). -# Arguments: -# APT syntax package, with or without version. -# Returns: -# Action syntax package, with or without version. -############################################################################### -function convert_action_to_apt_syntax_packages() { - echo ${1} | sed 's/:/=/g' -} - ############################################################################### # Execute the Debian install script. # Arguments: @@ -23,7 +11,7 @@ function convert_action_to_apt_syntax_packages() { # Filepath of the install script, otherwise an empty string. ############################################################################### function execute_install_script { - local package_name=$(basename ${2} | awk -F\: '{print $1}') + local package_name=$(basename ${2} | awk -F\= '{print $1}') local install_script_filepath=$(\ get_install_script_filepath "${1}" "${package_name}" "${3}") if test ! -z "${install_script_filepath}"; then @@ -67,7 +55,7 @@ function get_installed_packages { while read -r line; do # ${regex} should be unquoted since it isn't a literal. if [[ "${line}" =~ ${regex} ]]; then - dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[4]} " + dep_packages="${dep_packages}${BASH_REMATCH[1]}=${BASH_REMATCH[4]} " else log_err "Unable to parse package name and version from \"${line}\"" exit 2 @@ -89,13 +77,13 @@ function get_installed_packages { ############################################################################### function get_package_name_ver { local ORIG_IFS="${IFS}" - IFS=\: read name ver <<< "${1}" + IFS=\= read name ver <<< "${1}" + IFS="${ORIG_IFS}" # If version not found in the fully qualified package value. if test -z "${ver}"; then ver="$(grep "Version:" <<< "$(apt-cache show ${name})" | awk '{print $2}')" fi - echo "${name}" "${ver}" - IFS="${ORIG_IFS}" + echo "${name}" "${ver}" } ############################################################################### diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 328825f..326c3e9 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -56,8 +56,7 @@ log_empty_line versioned_packages="" log "Verifying packages..." for package in ${packages}; do - apt_syntax_package=$(convert_action_to_apt_syntax_packages ${package}) - if test ! "$(apt-cache show ${apt_syntax_package})"; then + if test ! "$(apt-cache show ${package})"; then echo "aborted" log "Package '${package}' not found." >&2 exit 5 @@ -80,7 +79,7 @@ log "- Normalized package list is '${normalized_versioned_packages}'." # Forces an update in cases where an accidental breaking change was introduced # and a global cache reset is required. -force_update_inc="0" +force_update_inc="1" value="${normalized_versioned_packages} @ ${version} ${force_update_inc}" log "- Value to hash is '${value}'." From 6d3c7590b1d0e03e4cbd50cafcec0287f2779301 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 23 Mar 2023 22:33:59 -0700 Subject: [PATCH 6/7] Test apt-cache show command wrt versioning. --- pre_cache_action.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 326c3e9..2b48b54 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -56,9 +56,10 @@ log_empty_line versioned_packages="" log "Verifying packages..." for package in ${packages}; do - if test ! "$(apt-cache show ${package})"; then + package_name=$(echo ${package} | awk -F\= '{print $1}') + if test ! "$(apt-cache show ${package_name})"; then echo "aborted" - log "Package '${package}' not found." >&2 + log "Package '${package_name}' not found." >&2 exit 5 fi read package_name package_ver < <(get_package_name_ver "${package}") From 0a4812359d55d4a52b47b38ed220062da36be666 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 23 Mar 2023 22:58:47 -0700 Subject: [PATCH 7/7] Revert package existence test. --- pre_cache_action.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 2b48b54..b487743 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -55,11 +55,10 @@ log_empty_line versioned_packages="" log "Verifying packages..." -for package in ${packages}; do - package_name=$(echo ${package} | awk -F\= '{print $1}') - if test ! "$(apt-cache show ${package_name})"; then +for package in ${packages}; do + if test ! "$(apt-cache show ${package})"; then echo "aborted" - log "Package '${package_name}' not found." >&2 + log "Package '${package}' not found." >&2 exit 5 fi read package_name package_ver < <(get_package_name_ver "${package}")