From e68df30bd6bff9a45d3de3069b72247c443d7e3c Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Sat, 26 Mar 2022 12:42:40 -0700 Subject: [PATCH 01/23] Experimental version. Cleaned up syntax and created common library functions. --- install_and_cache_pkgs.sh | 41 ++++++++++++++++++----------------- lib.sh | 23 ++++++++++++++++++++ pre_cache_action.sh | 45 ++++++++++++++++++--------------------- restore_pkgs.sh | 2 +- 4 files changed, 67 insertions(+), 44 deletions(-) create mode 100644 lib.sh diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index ca6c838..28b3858 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -3,22 +3,23 @@ # Fail on any error. set -e +# Include library. +script_dir="$(dirname -- "$(realpath -- "${0}")")" +source "${script_dir}/lib" + # Directory that holds the cached packages. cache_dir="${1}" # List of the packages to use. -packages="${@:2}" +input_packages="${@:2}" -# Sort these packages by name and split on commas. -packages=$(echo "${packages}" | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') - -# Remove extraneous spaces -packages="$(echo "${packages}" | sed 's/\s\+/ /g;s/^\s\+//g;s/\s\+$//g')" +# Trim commas, excess spaces, and sort. +packages="$(normalize_package_list "${input_packages}")" package_count=$(echo "${packages}" | wc -w) echo "Clean installing and caching ${package_count} package(s)." echo "Package list:" -for package in ${packages}; do +for package in "${packages}"; do echo "- ${package}" done @@ -28,21 +29,23 @@ echo "done." manifest= echo "Clean installing and caching ${package_count} packages..." -for package in ${packages}; do - echo "- ${package}" - echo -n " Installing..." +for package in "${packages}"; do + get_package_name_ver "${package}" # -> package_name, package_ver + package_deps="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')" - # Gather a list of all packages apt installs to make this package work - required_packages=$(apt-get install --dry-run --yes "${package}" | grep -Po "(?<=Inst )[^\s]+" || echo "${package}") - echo "Package ${package} installs the following packages: ${required_packages//$'\n'/, }" + echo "- ${package_name}" + echo " * Version: ${package_ver}" + echo " * Dependencies: ${package_deps}" + echo -n " * Installing..." + # Zero interaction while installing or upgrading the system via apt. sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null - echo "done." - for cache_package in ${required_packages}; do + for cache_package in "${package_deps}"; do cache_filepath="${cache_dir}/${cache_package}.tar.gz" - if [ ! -f "${cache_filepath}" ]; then - package_name="$(echo "${cache_package}" | cut -d"=" -f1)" + + if test ! -f "${cache_filepath}"; then + get_package_name_ver "${cache_package}" # -> package_name, package_ver echo -n " Caching ${package_name} to ${cache_filepath}..." # Pipe all package files (no folders) to Tar. dpkg -L "${package_name}" | @@ -60,6 +63,6 @@ echo "done." manifest_filepath="${cache_dir}/manifest.log" echo -n "Writing package manifest to ${manifest_filepath}..." -# Remove trailing comma. -echo ${manifest:0:-1} > ${manifest_filepath} +# Remove trailing comma and write to manifest file. +echo "${manifest:0:-1}" > "${manifest_filepath}" echo "done." \ No newline at end of file diff --git a/lib.sh b/lib.sh new file mode 100644 index 0000000..3802483 --- /dev/null +++ b/lib.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Sort these packages by name and split on commas. +function normalize_package_list { + stripped="$(echo \"${0}\" | sed 's/,//g')" + # Remove extraneous spaces at the middle, beginning, and end. + trimmed="$(echo \"${stripped}\" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" + echo "$(\"${trimmed}\" | sort)" +} + +# Split fully qualified package into name and version +function get_package_name_ver { + IFS=\= read name ver <<< "${0}" + # If version not found in the fully qualified package value. + if test -z "${ver}"; then + ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')" + fi + echo 'package_name="${name}"; package_ver="${ver}"' +} + +function blah { + > /dev/null 2>&1 +} \ No newline at end of file diff --git a/pre_cache_action.sh b/pre_cache_action.sh index d9dd664..d1899f5 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -1,5 +1,9 @@ #!/bin/bash +# Include library. +script_dir="$(dirname -- "$(realpath -- "${0}")")" +source "${script_dir}/lib" + # Directory that holds the cached packages. cache_dir="${1}" @@ -7,22 +11,23 @@ cache_dir="${1}" cache_version="${2}" # List of the packages to use. -packages="${@:3}" +input_packages="${@:3}" -# Sort these packages by name and split on commas. -packages=$(echo "${packages}" | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') +# Trim commas, excess spaces, and sort. +packages="$(normalize_package_list "${input_packages}")" # Create cache directory so artifacts can be saved. mkdir -p "${cache_dir}" echo -n "Validating action arguments (version='${cache_version}', packages='${packages}')..."; -if echo "${cache_version}" | grep -q " " > /dev/null; then +if grep -q " " <<< "${cache_version}"; then echo "aborted." echo "Version value '${cache_version}' cannot contain spaces." >&2 exit 1 fi -if [ "${packages}" == "" ]; then +# Is length of string zero? +if test -z "${packages}"; then echo "aborted." echo "Packages argument cannot be empty." >&2 exit 2 @@ -33,24 +38,16 @@ echo -n "Updating APT package list..." sudo apt-get update > /dev/null echo "done." +versioned_packages="" echo -n "Verifying packages..." -for package in ${packages}; do - if echo "${package}" | grep -q "="; then - package_name=$(echo "${package}" | cut -d "=" -f1) - package_ver=$(echo "${package}" | cut -d "=" -f2) - else - package_name="${package}" - fi - apt_show=$(apt show "${package}") - if echo ${apt_show} | grep -qi "No packages found" > /dev/null; then +for package in ${packages}; do + if test ! "$(apt show "${package}")"; then echo "aborted." echo "Package '${package}' not found." >&2 exit 3 fi - if [ -z "${package_ver}" ]; then - package_ver=$(echo "${apt_show}" | grep -Po "(?<=Version: )[^\s]+") - fi - package_list="${package_list} ${package_name}=${package_ver}" + get_package_name_ver "${package}" # -> package_name, package_ver + versioned_packages="${versioned_packages} ${package_name}=${package_ver}" done echo "done." @@ -59,15 +56,15 @@ set -e echo "Creating cache key..." -# Remove extraneous spaces -package_list="$(echo "${package_list}" | sed 's/\s\+/ /g;s/^\s\+//g;s/\s\+$//g')" -echo "- Normalized package list is '$package_list'." +# TODO Can we prove this will happen again? +normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")" +echo "- Normalized package list is '${normalized_versioned_packages}'." -value=$(echo "${package_list} @ ${cache_version}") +value="$(echo "${normalized_versioned_packages} @ ${cache_version}")" echo "- Value to hash is '${value}'." -key=$(echo "${value}" | md5sum | /bin/cut -f1 -d' ') -echo "- Value hashed as '$key'." +key="$(echo "${value}" | md5sum | /bin/cut -f1 -d' ')" +echo "- Value hashed as '${key}'." echo "done." diff --git a/restore_pkgs.sh b/restore_pkgs.sh index 4fdbcc6..87ee131 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -17,7 +17,7 @@ for cache_filepath in ${cache_filepaths}; do done echo "Reading from manifest..." -for logline in $(cat "${cache_dir}/manifest.log" | tr ',' '\n' ); do +for logline in "$(cat "${cache_dir}/manifest.log" | tr ',' '\n' )"; do echo "- $(echo "${logline}" | tr ':' ' ')" done echo "done." From 9568dc9783b4f9e973b956e618f5895fbda6b199 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Sat, 26 Mar 2022 12:48:16 -0700 Subject: [PATCH 02/23] Change include command. --- install_and_cache_pkgs.sh | 2 +- pre_cache_action.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 28b3858..337628f 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -5,7 +5,7 @@ set -e # Include library. script_dir="$(dirname -- "$(realpath -- "${0}")")" -source "${script_dir}/lib" +source "${script_dir}/lib.sh" # Directory that holds the cached packages. cache_dir="${1}" diff --git a/pre_cache_action.sh b/pre_cache_action.sh index d1899f5..8c8f992 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -2,7 +2,7 @@ # Include library. script_dir="$(dirname -- "$(realpath -- "${0}")")" -source "${script_dir}/lib" +source "${script_dir}/lib.sh" # Directory that holds the cached packages. cache_dir="${1}" From 17cdbf8735bb6dbb7f1458268cb2cfed9111439e Mon Sep 17 00:00:00 2001 From: Andrew Walsh Date: Fri, 3 Jun 2022 20:30:40 -0700 Subject: [PATCH 03/23] Revert "Cache non explicitly listed packages" --- action.yml | 4 --- install_and_cache_pkgs.sh | 66 +++++++++++++++------------------------ post_cache_action.sh | 14 ++++----- pre_cache_action.sh | 61 +++++++++++++----------------------- restore_pkgs.sh | 28 ++++++++--------- 5 files changed, 69 insertions(+), 104 deletions(-) diff --git a/action.yml b/action.yml index 38d4e13..e717de5 100644 --- a/action.yml +++ b/action.yml @@ -25,9 +25,6 @@ outputs: # This compound expression is needed because lhs can be empty. # Need to output true and false instead of true and nothing. value: ${{ steps.load-cache.outputs.cache-hit || false }} - cache-key: - description: 'The key where the cache is stored with packages, their versions, and the cache version (i.e. package=version package=version @ 1).' - value: ${{ steps.pre-cache.outputs.cache-key }} package-version-list: description: 'The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. ::).' value: ${{ steps.post-cache.outputs.package-version-list }} @@ -42,7 +39,6 @@ runs: "${{ inputs.version }}" \ ${{ inputs.packages }} echo "CACHE_KEY=$(cat ~/cache-apt-pkgs/cache_key.md5)" >> $GITHUB_ENV - echo "::set-output name=cache-key::$(cat ~/cache-apt-pkgs/cache_key.md5)" shell: bash - id: load-cache diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index ca6c838..bbf54dc 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -4,62 +4,48 @@ set -e # Directory that holds the cached packages. -cache_dir="${1}" +cache_dir=$1 # List of the packages to use. packages="${@:2}" -# Sort these packages by name and split on commas. -packages=$(echo "${packages}" | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') - -# Remove extraneous spaces -packages="$(echo "${packages}" | sed 's/\s\+/ /g;s/^\s\+//g;s/\s\+$//g')" - -package_count=$(echo "${packages}" | wc -w) -echo "Clean installing and caching ${package_count} package(s)." +package_count=$(echo $packages | wc -w) +echo "Clean installing and caching $package_count package(s)." echo "Package list:" -for package in ${packages}; do - echo "- ${package}" +for package in $packages; do + echo "- $package" done echo -n "Updating APT package list..." sudo apt-get update > /dev/null echo "done." -manifest= -echo "Clean installing and caching ${package_count} packages..." -for package in ${packages}; do - echo "- ${package}" +echo "Clean installing and caching $(echo $packages | wc -w) packages..." +for package in $packages; do + cache_filepath=$cache_dir/$package.tar.gz + + echo "- $package" echo -n " Installing..." - - # Gather a list of all packages apt installs to make this package work - required_packages=$(apt-get install --dry-run --yes "${package}" | grep -Po "(?<=Inst )[^\s]+" || echo "${package}") - echo "Package ${package} installs the following packages: ${required_packages//$'\n'/, }" - sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null - + sudo apt-get --yes install $package > /dev/null echo "done." - for cache_package in ${required_packages}; do - cache_filepath="${cache_dir}/${cache_package}.tar.gz" - if [ ! -f "${cache_filepath}" ]; then - package_name="$(echo "${cache_package}" | cut -d"=" -f1)" - echo -n " Caching ${package_name} to ${cache_filepath}..." - # Pipe all package files (no folders) to Tar. - dpkg -L "${package_name}" | - while IFS= read -r f; do - if test -f $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows - done | - xargs tar -czf "${cache_filepath}" -C / - echo "done." - # Add package to manifest - manifest="${manifest}${package_name}:$(dpkg -s "${package_name}" | grep Version | awk '{print $2}')," - fi - done + echo -n " Caching to $cache_filepath..." + # Pipe all package files (no folders) to Tar. + dpkg -L $package | + while IFS= read -r f; do + if test -f $f; then echo ${f:1}; fi; #${f:1} removes the leading slash that Tar disallows + done | + xargs tar -czf $cache_filepath -C / + echo "done." done echo "done." -manifest_filepath="${cache_dir}/manifest.log" -echo -n "Writing package manifest to ${manifest_filepath}..." +manifest_filepath="$cache_dir/manifest.log" +echo -n "Writing package manifest to $manifest_filepath..." +manifest= +for package in $packages; do + manifest=$manifest$package:$(dpkg -s $package | grep Version | awk '{print $2}'), +done # Remove trailing comma. -echo ${manifest:0:-1} > ${manifest_filepath} +echo ${manifest:0:-1} > $manifest_filepath echo "done." \ No newline at end of file diff --git a/post_cache_action.sh b/post_cache_action.sh index 0bb091f..5503013 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -4,23 +4,23 @@ set -e # Directory that holds the cached packages. -cache_dir="${1}" +cache_dir=$1 # Root directory to untar the cached packages to. # Typically filesystem root '/' but can be changed for testing. -cache_restore_root="${2}" +cache_restore_root=$2 # Indicates that the cache was found. -cache_hit="${3}" +cache_hit=$3 # List of the packages to use. packages="${@:4}" -script_dir="$(dirname -- "$(realpath -- "${0}")")" +script_dir=$(dirname $0) -if [ "${cache_hit}" == true ]; then - "${script_dir}/restore_pkgs.sh" ~/cache-apt-pkgs "${cache_restore_root}" +if [ "$cache_hit" == true ]; then + $script_dir/restore_pkgs.sh ~/cache-apt-pkgs $cache_restore_root else - "${script_dir}/install_and_cache_pkgs.sh" ~/cache-apt-pkgs "${packages}" + $script_dir/install_and_cache_pkgs.sh ~/cache-apt-pkgs $packages fi echo "" diff --git a/pre_cache_action.sh b/pre_cache_action.sh index d9dd664..51a5b78 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -1,56 +1,39 @@ #!/bin/bash # Directory that holds the cached packages. -cache_dir="${1}" +cache_dir=$1 # Version of the cache to create or load. -cache_version="${2}" +version=$2 # List of the packages to use. -packages="${@:3}" - -# Sort these packages by name and split on commas. -packages=$(echo "${packages}" | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') +packages=${@:3} # Create cache directory so artifacts can be saved. -mkdir -p "${cache_dir}" +mkdir -p $cache_dir -echo -n "Validating action arguments (version='${cache_version}', packages='${packages}')..."; - -if echo "${cache_version}" | grep -q " " > /dev/null; then +echo -n "Validating action arguments (version='$version', packages='$packages')..."; +echo $version | grep -o " " > /dev/null +if [ $? -eq 0 ]; then echo "aborted." - echo "Version value '${cache_version}' cannot contain spaces." >&2 + echo "Version value '$version' cannot contain spaces." >&2 exit 1 fi -if [ "${packages}" == "" ]; then +if [ "$packages" == "" ]; then echo "aborted." echo "Packages argument cannot be empty." >&2 exit 2 fi echo "done." -echo -n "Updating APT package list..." -sudo apt-get update > /dev/null -echo "done." - echo -n "Verifying packages..." -for package in ${packages}; do - if echo "${package}" | grep -q "="; then - package_name=$(echo "${package}" | cut -d "=" -f1) - package_ver=$(echo "${package}" | cut -d "=" -f2) - else - package_name="${package}" - fi - apt_show=$(apt show "${package}") - if echo ${apt_show} | grep -qi "No packages found" > /dev/null; then +for package in $packages; do + apt-cache search ^$package$ | grep $package > /dev/null + if [ $? -ne 0 ]; then echo "aborted." - echo "Package '${package}' not found." >&2 + echo "Package '$package' not found." >&2 exit 3 fi - if [ -z "${package_ver}" ]; then - package_ver=$(echo "${apt_show}" | grep -Po "(?<=Version: )[^\s]+") - fi - package_list="${package_list} ${package_name}=${package_ver}" done echo "done." @@ -59,18 +42,18 @@ set -e echo "Creating cache key..." -# Remove extraneous spaces -package_list="$(echo "${package_list}" | sed 's/\s\+/ /g;s/^\s\+//g;s/\s\+$//g')" -echo "- Normalized package list is '$package_list'." +# Remove package delimiters, sort (requires newline) and then convert back to inline list. +normalized_list=$(echo $packages | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') +echo "- Normalized package list is '$normalized_list'." -value=$(echo "${package_list} @ ${cache_version}") -echo "- Value to hash is '${value}'." +value=$(echo $normalized_list @ $version) +echo "- Value to hash is '$value'." -key=$(echo "${value}" | md5sum | /bin/cut -f1 -d' ') +key=$(echo $value | md5sum | /bin/cut -f1 -d' ') echo "- Value hashed as '$key'." echo "done." -key_filepath="${cache_dir}/cache_key.md5" -echo "${key}" > "${key_filepath}" -echo "Hash value written to ${key_filepath}" +key_filepath="$cache_dir/cache_key.md5" +echo $key > $key_filepath +echo "Hash value written to $key_filepath" diff --git a/restore_pkgs.sh b/restore_pkgs.sh index 4fdbcc6..ab87879 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -4,31 +4,31 @@ set -e # Directory that holds the cached packages. -cache_dir="${1}" +cache_dir=$1 # Root directory to untar the cached packages to. # Typically filesystem root '/' but can be changed for testing. -cache_restore_root="${2}" +cache_restore_root=$2 -cache_filepaths="$(ls -1 "${cache_dir}" | sort)" -echo "Found $(echo "${cache_filepaths}" | wc -w) files in the cache." -for cache_filepath in ${cache_filepaths}; do - echo "- $(basename "${cache_filepath}")" +cache_filepaths=$(ls -1 $cache_dir | sort) +echo "Found $(echo $cache_filepaths | wc -w) files in the cache." +for cache_filepath in $cache_filepaths; do + echo "- $(basename $cache_filepath)" done echo "Reading from manifest..." -for logline in $(cat "${cache_dir}/manifest.log" | tr ',' '\n' ); do - echo "- $(echo "${logline}" | tr ':' ' ')" +for logline in $(cat $cache_dir/manifest.log | tr ',' '\n' ); do + echo "- $(echo $logline | tr ':' ' ')" done echo "done." # Only search for archived results. Manifest and cache key also live here. -cache_pkg_filepaths=$(ls -1 ${cache_dir}/*.tar.gz | sort) -cache_pkg_filecount=$(echo "${cache_pkg_filepaths}" | wc -w) -echo "Restoring ${cache_pkg_filecount} packages from cache..." -for cache_pkg_filepath in ${cache_pkg_filepaths}; do - echo -n "- $(basename "${cache_pkg_filepath}") restoring..." - sudo tar -xf ${cache_pkg_filepath} -C ${cache_restore_root} > /dev/null +cache_pkg_filepaths=$(ls -1 $cache_dir/*.tar.gz | sort) +cache_pkg_filecount=$(echo $cache_pkg_filepaths | wc -w) +echo "Restoring $cache_pkg_filecount packages from cache..." +for cache_pkg_filepath in $cache_pkg_filepaths; do + echo -n "- $(basename $cache_pkg_filepath) restoring..." + sudo tar -xf $cache_pkg_filepath -C $cache_restore_root > /dev/null echo "done." done echo "done." From 1ca2ac1a4cbfe99f76f137054e3a5792b593356b Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 00:30:31 -0700 Subject: [PATCH 04/23] Standardize syntax and script sourcing. --- install_and_cache_pkgs.sh | 2 +- lib.sh | 4 ---- post_cache_action.sh | 12 ++++++------ pre_cache_action.sh | 18 +++++++++--------- restore_pkgs.sh | 24 ++++++++++++------------ 5 files changed, 28 insertions(+), 32 deletions(-) mode change 100644 => 100755 lib.sh diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index e4f6c2c..f3cdfe5 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -8,7 +8,7 @@ script_dir="$(dirname -- "$(realpath -- "${0}")")" source "${script_dir}/lib.sh" # Directory that holds the cached packages. -cache_dir=$1 +cache_dir="${1}" # List of the packages to use. input_packages="${@:2}" diff --git a/lib.sh b/lib.sh old mode 100644 new mode 100755 index 3802483..2a31aac --- a/lib.sh +++ b/lib.sh @@ -17,7 +17,3 @@ function get_package_name_ver { fi echo 'package_name="${name}"; package_ver="${ver}"' } - -function blah { - > /dev/null 2>&1 -} \ No newline at end of file diff --git a/post_cache_action.sh b/post_cache_action.sh index 5503013..56fda54 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -4,23 +4,23 @@ set -e # Directory that holds the cached packages. -cache_dir=$1 +cache_dir="${1}" # Root directory to untar the cached packages to. # Typically filesystem root '/' but can be changed for testing. -cache_restore_root=$2 +cache_restore_root="${2}" # Indicates that the cache was found. -cache_hit=$3 +cache_hit="${3}" # List of the packages to use. packages="${@:4}" -script_dir=$(dirname $0) +script_dir="$(dirname -- "$(realpath -- "${0}")")" if [ "$cache_hit" == true ]; then - $script_dir/restore_pkgs.sh ~/cache-apt-pkgs $cache_restore_root + ${script_dir}/restore_pkgs.sh ~/cache-apt-pkgs ${cache_restore_root} else - $script_dir/install_and_cache_pkgs.sh ~/cache-apt-pkgs $packages + ${script_dir}/install_and_cache_pkgs.sh ~/cache-apt-pkgs ${packages} fi echo "" diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 05d2fbe..4bd63d4 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -5,10 +5,10 @@ script_dir="$(dirname -- "$(realpath -- "${0}")")" source "${script_dir}/lib.sh" # Directory that holds the cached packages. -cache_dir=$1 +cache_dir=${1} # Version of the cache to create or load. -version=$2 +version=${2} # List of the packages to use. input_packages="${@:3}" @@ -17,12 +17,12 @@ input_packages="${@:3}" packages="$(normalize_package_list "${input_packages}")" # Create cache directory so artifacts can be saved. -mkdir -p $cache_dir +mkdir -p ${cache_dir} -echo -n "Validating action arguments (version='$version', packages='$packages')..."; +echo -n "Validating action arguments (version='${version}', packages='${packages}')..."; if grep -q " " <<< "${cache_version}"; then echo "aborted." - echo "Version value '$version' cannot contain spaces." >&2 + echo "Version value '${version}' cannot contain spaces." >&2 exit 1 fi @@ -39,7 +39,7 @@ echo -n "Verifying packages..." for package in ${packages}; do if test ! "$(apt show "${package}")"; then echo "aborted." - echo "Package '$package' not found." >&2 + echo "Package '${package}' not found." >&2 exit 3 fi get_package_name_ver "${package}" # -> package_name, package_ver @@ -64,6 +64,6 @@ echo "- Value hashed as '${key}'." echo "done." -key_filepath="$cache_dir/cache_key.md5" -echo $key > $key_filepath -echo "Hash value written to $key_filepath" +key_filepath="${cache_dir}/cache_key.md5" +echo ${key} > ${key_filepath} +echo "Hash value written to ${key_filepath}" diff --git a/restore_pkgs.sh b/restore_pkgs.sh index 4c710f3..3cee703 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -4,16 +4,16 @@ set -e # Directory that holds the cached packages. -cache_dir=$1 +cache_dir="${1}" # Root directory to untar the cached packages to. # Typically filesystem root '/' but can be changed for testing. -cache_restore_root=$2 +cache_restore_root="${2}" -cache_filepaths=$(ls -1 $cache_dir | sort) -echo "Found $(echo $cache_filepaths | wc -w) files in the cache." -for cache_filepath in $cache_filepaths; do - echo "- $(basename $cache_filepath)" +cache_filepaths=$(ls -1 ${cache_dir} | sort) +echo "Found $(echo ${cache_filepaths} | wc -w) files in the cache." +for cache_filepath in ${cache_filepaths}; do + echo "- $(basename ${cache_filepath})" done echo "Reading from manifest..." @@ -23,12 +23,12 @@ done echo "done." # Only search for archived results. Manifest and cache key also live here. -cache_pkg_filepaths=$(ls -1 $cache_dir/*.tar.gz | sort) -cache_pkg_filecount=$(echo $cache_pkg_filepaths | wc -w) -echo "Restoring $cache_pkg_filecount packages from cache..." -for cache_pkg_filepath in $cache_pkg_filepaths; do - echo -n "- $(basename $cache_pkg_filepath) restoring..." - sudo tar -xf $cache_pkg_filepath -C $cache_restore_root > /dev/null +cache_pkg_filepaths=$(ls -1 ${cache_dir}/*.tar.gz | sort) +cache_pkg_filecount=$(echo ${cache_pkg_filepaths} | wc -w) +echo "Restoring ${cache_pkg_filecount} packages from cache..." +for cache_pkg_filepath in ${cache_pkg_filepaths}; do + echo -n "- $(basename ${cache_pkg_filepath}) restoring..." + sudo tar -xf ${cache_pkg_filepath} -C ${cache_restore_root} > /dev/null echo "done." done echo "done." From c961be0ea58da1e5eb6b11e28e22fd1f83665613 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 00:51:24 -0700 Subject: [PATCH 05/23] Fix lib arg ref and quoting behavior. --- lib.sh | 8 ++++---- pre_cache_action.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib.sh b/lib.sh index 2a31aac..a8011a2 100755 --- a/lib.sh +++ b/lib.sh @@ -2,15 +2,15 @@ # Sort these packages by name and split on commas. function normalize_package_list { - stripped="$(echo \"${0}\" | sed 's/,//g')" + stripped=$(echo "${1}" | sed 's/,//g') # Remove extraneous spaces at the middle, beginning, and end. - trimmed="$(echo \"${stripped}\" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" - echo "$(\"${trimmed}\" | sort)" + trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" + echo "$(echo "${trimmed}" | sort)" } # Split fully qualified package into name and version function get_package_name_ver { - IFS=\= read name ver <<< "${0}" + IFS=\= read name ver <<< "${1}" # If version not found in the fully qualified package value. if test -z "${ver}"; then ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')" diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 4bd63d4..1b0c70c 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -5,10 +5,10 @@ script_dir="$(dirname -- "$(realpath -- "${0}")")" source "${script_dir}/lib.sh" # Directory that holds the cached packages. -cache_dir=${1} +cache_dir="${1}" # Version of the cache to create or load. -version=${2} +version="${2}" # List of the packages to use. input_packages="${@:3}" From 45b8861f3a7c4da4be7cdf6e950eab8ba54aeea6 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 01:06:32 -0700 Subject: [PATCH 06/23] Fix package name ver function output. --- lib.sh | 2 +- pre_cache_action.sh | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib.sh b/lib.sh index a8011a2..975fc3f 100755 --- a/lib.sh +++ b/lib.sh @@ -15,5 +15,5 @@ function get_package_name_ver { if test -z "${ver}"; then ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')" fi - echo 'package_name="${name}"; package_ver="${ver}"' + echo ""${name}"="${ver}"" } diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 1b0c70c..84d9868 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -41,9 +41,8 @@ for package in ${packages}; do echo "aborted." echo "Package '${package}' not found." >&2 exit 3 - fi - get_package_name_ver "${package}" # -> package_name, package_ver - versioned_packages="${versioned_packages} ${package_name}=${package_ver}" + fi + versioned_packages=""${versioned_packages}" "$(get_package_name_ver "${package}")"" done echo "done." From e6baec0bc6c0871eaee63b59fa4e0b64bf575c6b Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 01:17:13 -0700 Subject: [PATCH 07/23] Fix cache version ref and ver function output. --- lib.sh | 2 +- pre_cache_action.sh | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib.sh b/lib.sh index 975fc3f..0c9ddd1 100755 --- a/lib.sh +++ b/lib.sh @@ -15,5 +15,5 @@ function get_package_name_ver { if test -z "${ver}"; then ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')" fi - echo ""${name}"="${ver}"" + echo "${name}" "${ver}" } diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 84d9868..2f26954 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -20,7 +20,7 @@ packages="$(normalize_package_list "${input_packages}")" mkdir -p ${cache_dir} echo -n "Validating action arguments (version='${version}', packages='${packages}')..."; -if grep -q " " <<< "${cache_version}"; then +if grep -q " " <<< "${version}"; then echo "aborted." echo "Version value '${version}' cannot contain spaces." >&2 exit 1 @@ -41,8 +41,9 @@ for package in ${packages}; do echo "aborted." echo "Package '${package}' not found." >&2 exit 3 - fi - versioned_packages=""${versioned_packages}" "$(get_package_name_ver "${package}")"" + fi + read package_name package_ver < <(get_package_name_ver "${package}") # -> package_name, package_ver + versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}"" done echo "done." @@ -55,7 +56,7 @@ echo "Creating cache key..." normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")" echo "- Normalized package list is '${normalized_versioned_packages}'." -value="$(echo "${normalized_versioned_packages} @ ${cache_version}")" +value="$(echo "${normalized_versioned_packages} @ ${version}")" echo "- Value to hash is '${value}'." key="$(echo "${value}" | md5sum | /bin/cut -f1 -d' ')" From 686ae5cbb1a074a8a2968d11b0c31fb16fc1c023 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 01:27:20 -0700 Subject: [PATCH 08/23] Fix package enumeration. --- install_and_cache_pkgs.sh | 10 +++++----- pre_cache_action.sh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index f3cdfe5..81096ed 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -14,12 +14,12 @@ cache_dir="${1}" input_packages="${@:2}" # Trim commas, excess spaces, and sort. -packages="$(normalize_package_list "${input_packages}")" +normalized_packages="$(normalize_package_list "${input_packages}")" -package_count=$(echo "${packages}" | wc -w) +package_count=$(echo "${normalized_packages}" | wc -w) echo "Clean installing and caching ${package_count} package(s)." echo "Package list:" -for package in "${packages}"; do +for package in ${normalized_packages}; do echo "- ${package}" done @@ -29,8 +29,8 @@ echo "done." manifest="" echo "Clean installing and caching ${package_count} packages..." -for package in "${packages}"; do - get_package_name_ver "${package}" # -> package_name, package_ver +for package in ${normalized_packages}; do + read package_name package_ver < <(get_package_name_ver "${package}") package_deps="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')" echo "- ${package_name}" diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 2f26954..ec33dae 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -42,7 +42,7 @@ for package in ${packages}; do echo "Package '${package}' not found." >&2 exit 3 fi - read package_name package_ver < <(get_package_name_ver "${package}") # -> package_name, package_ver + read package_name package_ver < <(get_package_name_ver "${package}") versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}"" done echo "done." From cb0990bfa293a145334e634cc744ccffc7364273 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 01:35:13 -0700 Subject: [PATCH 09/23] Fix package enumeration. --- post_cache_action.sh | 2 +- restore_pkgs.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/post_cache_action.sh b/post_cache_action.sh index 56fda54..8440aa3 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -19,7 +19,7 @@ packages="${@:4}" script_dir="$(dirname -- "$(realpath -- "${0}")")" if [ "$cache_hit" == true ]; then - ${script_dir}/restore_pkgs.sh ~/cache-apt-pkgs ${cache_restore_root} + ${script_dir}/restore_pkgs.sh ~/cache-apt-pkgs "${cache_restore_root}" else ${script_dir}/install_and_cache_pkgs.sh ~/cache-apt-pkgs ${packages} fi diff --git a/restore_pkgs.sh b/restore_pkgs.sh index 3cee703..e7b2e10 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -10,25 +10,25 @@ cache_dir="${1}" # Typically filesystem root '/' but can be changed for testing. cache_restore_root="${2}" -cache_filepaths=$(ls -1 ${cache_dir} | sort) +cache_filepaths="$(ls -1 "${cache_dir}" | sort)" echo "Found $(echo ${cache_filepaths} | wc -w) files in the cache." for cache_filepath in ${cache_filepaths}; do - echo "- $(basename ${cache_filepath})" + echo "- "$(basename ${cache_filepath})"" done echo "Reading from manifest..." -for logline in "$(cat "${cache_dir}/manifest.log" | tr ',' '\n' )"; do +for logline in $(cat "${cache_dir}/manifest.log" | tr ',' '\n' ); do echo "- $(echo "${logline}" | tr ':' ' ')" done echo "done." # Only search for archived results. Manifest and cache key also live here. -cache_pkg_filepaths=$(ls -1 ${cache_dir}/*.tar.gz | sort) +cache_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort) cache_pkg_filecount=$(echo ${cache_pkg_filepaths} | wc -w) echo "Restoring ${cache_pkg_filecount} packages from cache..." for cache_pkg_filepath in ${cache_pkg_filepaths}; do - echo -n "- $(basename ${cache_pkg_filepath}) restoring..." - sudo tar -xf ${cache_pkg_filepath} -C ${cache_restore_root} > /dev/null + echo -n "- $(basename "${cache_pkg_filepath}") restoring..." + sudo tar -xf "${cache_pkg_filepath}" -C "${cache_restore_root}" > /dev/null echo "done." done echo "done." From bc3229644bf719a4d335c8ba482683c248c74f65 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 01:41:00 -0700 Subject: [PATCH 10/23] Fix another package enumeration. --- install_and_cache_pkgs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 81096ed..21ba73e 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -41,7 +41,7 @@ for package in ${normalized_packages}; do sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null echo "done." - for cache_package in "${package_deps}"; do + for cache_package in ${package_deps}; do cache_filepath="${cache_dir}/${cache_package}.tar.gz" if test ! -f "${cache_filepath}"; then From 903a8a32ff59da642d5a1439c62b26f0ae6cb15d Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 02:10:23 -0700 Subject: [PATCH 11/23] Fix read package name and version call. --- install_and_cache_pkgs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 21ba73e..6348403 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -45,7 +45,7 @@ for package in ${normalized_packages}; do cache_filepath="${cache_dir}/${cache_package}.tar.gz" if test ! -f "${cache_filepath}"; then - get_package_name_ver "${cache_package}" # -> package_name, package_ver + read package_name package_ver < <(get_package_name_ver "${cache_package}") echo -n " Caching ${package_name} to ${cache_filepath}..." # Pipe all package files (no folders) to Tar. dpkg -L "${package_name}" | From a91bf7b286265e7c644531f582c81bc0a1085bc9 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 06:18:19 -0700 Subject: [PATCH 12/23] Create a new output for all packages including dependencies. --- README.md | 4 ++-- action.yml | 8 +++++-- install_and_cache_pkgs.sh | 50 ++++++++++++++++++++++++++------------- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 927e93a..972ce8a 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ Create a workflow `.yml` file in your repositories `.github/workflows` directory ### Outputs * `cache-hit` - A boolean value to indicate a cache was found for the packages requested. -* `package-version-list` - The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. \:,\:\,...). - +* `package-version-list` - The main requested packages and versions that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. \:,\:\,...). +* `all-package-version-list` - All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. \:,\:\,...). ### Cache scopes diff --git a/action.yml b/action.yml index e717de5..c20f70a 100644 --- a/action.yml +++ b/action.yml @@ -26,7 +26,10 @@ outputs: # Need to output true and false instead of true and nothing. value: ${{ steps.load-cache.outputs.cache-hit || false }} package-version-list: - description: 'The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. ::).' + description: 'The main requested packages and versions that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. ::).' + value: ${{ steps.post-cache.outputs.package-version-list }} + all-package-version-list: + description: 'All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. ::).' value: ${{ steps.post-cache.outputs.package-version-list }} runs: @@ -54,5 +57,6 @@ runs: / \ "${{ steps.load-cache.outputs.cache-hit }}" \ ${{ inputs.packages }} - echo "::set-output name=package-version-list::$(cat ~/cache-apt-pkgs/manifest.log)" + echo "::set-output name=package-version-list::$(cat ~/cache-apt-pkgs/manifest_main.log)" + echo "::set-output name=all-package-version-list::$(cat ~/cache-apt-pkgs/manifest_all.log)" shell: bash diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 6348403..0d741d9 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -16,7 +16,7 @@ input_packages="${@:2}" # Trim commas, excess spaces, and sort. normalized_packages="$(normalize_package_list "${input_packages}")" -package_count=$(echo "${normalized_packages}" | wc -w) +package_count=$(wc -w <<< "${normalized_packages}") echo "Clean installing and caching ${package_count} package(s)." echo "Package list:" for package in ${normalized_packages}; do @@ -27,42 +27,58 @@ echo -n "Updating APT package list..." sudo apt-get update > /dev/null echo "done." -manifest="" +# Strictly contains the requested packages. +manifest_main="" +# Contains all packages including dependencies. +manifest_all="" + echo "Clean installing and caching ${package_count} packages..." for package in ${normalized_packages}; do - read package_name package_ver < <(get_package_name_ver "${package}") - package_deps="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')" + read package_name package_ver < <(get_package_name_ver "${package}") + + # Comma delimited name:ver pairs in the main requested packages manifest. + manifest_main="${manifest_main}${package_name}:${package_ver}," + + all_packages="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')" + dep_packages="$(grep -v "${package_name}" <<< ${all_packages})" echo "- ${package_name}" echo " * Version: ${package_ver}" - echo " * Dependencies: ${package_deps}" + echo " * Dependencies: ${dep_packages}" echo -n " * Installing..." # Zero interaction while installing or upgrading the system via apt. sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null echo "done." - for cache_package in ${package_deps}; do + for cache_package in ${all_packages}; do cache_filepath="${cache_dir}/${cache_package}.tar.gz" if test ! -f "${cache_filepath}"; then - read package_name package_ver < <(get_package_name_ver "${cache_package}") - echo -n " Caching ${package_name} to ${cache_filepath}..." + read cache_package_name cache_package_ver < <(get_package_name_ver "${cache_package}") + echo -n " Caching ${cache_package_name} to ${cache_filepath}..." # Pipe all package files (no folders) to Tar. - dpkg -L "${package_name}" | + dpkg -L "${cache_package_name}" | while IFS= read -r f; do if test -f $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows done | xargs tar -czf "${cache_filepath}" -C / - echo "done." - # Add package to manifest - manifest="${manifest}${package_name}:$(dpkg -s "${package_name}" | grep Version | awk '{print $2}')," + echo "done." fi - done + + # Comma delimited name:ver pairs in the all packages manifest. + manifest_all="${manifest_all}${cache_package_name}:${cache_package_ver}," + done done echo "done." -manifest_filepath="${cache_dir}/manifest.log" -echo -n "Writing package manifest to ${manifest_filepath}..." -# Remove trailing comma and write to manifest file. -echo "${manifest:0:-1}" > "${manifest_filepath}" +manifest_all_filepath="${cache_dir}/manifest_all.log" +echo -n "Writing all packages manifest to ${manifest_all_filepath}..." +# Remove trailing comma and write to manifest_all file. +echo "${manifest_all:0:-1}" > "${manifest_all_filepath}" +echo "done." + +manifest_main_filepath="${cache_dir}/manifest_main.log" +echo -n "Writing main requested packages manifest to ${manifest_main_filepath}..." +# Remove trailing comma and write to manifest_main file. +echo "${manifest_main:0:-1}" > "${manifest_main_filepath}" echo "done." \ No newline at end of file From f9b0016d025185cfddbff87dbb6645eb6ced7963 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 06:28:52 -0700 Subject: [PATCH 13/23] Inline dependency reporting in log. --- install_and_cache_pkgs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 0d741d9..1b49407 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -40,11 +40,11 @@ for package in ${normalized_packages}; do manifest_main="${manifest_main}${package_name}:${package_ver}," all_packages="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')" - dep_packages="$(grep -v "${package_name}" <<< ${all_packages})" + dep_packages="$(echo ${dep_packages} | grep -v "${package_name}" | tr '\n' ,)" echo "- ${package_name}" echo " * Version: ${package_ver}" - echo " * Dependencies: ${dep_packages}" + echo " * Dependencies: ${dep_packages:0:-1}" echo -n " * Installing..." # Zero interaction while installing or upgrading the system via apt. sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null From f5249e37b07a3284b58eb02d06988df145ed30be Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 06:29:47 -0700 Subject: [PATCH 14/23] Update caching log lines. --- install_and_cache_pkgs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 1b49407..e6533a1 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -55,7 +55,7 @@ for package in ${normalized_packages}; do if test ! -f "${cache_filepath}"; then read cache_package_name cache_package_ver < <(get_package_name_ver "${cache_package}") - echo -n " Caching ${cache_package_name} to ${cache_filepath}..." + echo -n " * Caching ${cache_package_name} to ${cache_filepath}..." # Pipe all package files (no folders) to Tar. dpkg -L "${cache_package_name}" | while IFS= read -r f; do From 843a78dbf3681d50a16477ca2eea8556a077104a Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 06:37:31 -0700 Subject: [PATCH 15/23] Fix manifest output on restore. --- restore_pkgs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/restore_pkgs.sh b/restore_pkgs.sh index e7b2e10..655490b 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -16,8 +16,8 @@ for cache_filepath in ${cache_filepaths}; do echo "- "$(basename ${cache_filepath})"" done -echo "Reading from manifest..." -for logline in $(cat "${cache_dir}/manifest.log" | tr ',' '\n' ); do +echo "Reading from main requested packages manifest..." +for logline in $(cat "${cache_dir}/manifest_main.log" | tr ',' '\n' ); do echo "- $(echo "${logline}" | tr ':' ' ')" done echo "done." From eb177a2a2b2cc7c73fac60fdecce070222db12b9 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 06:43:45 -0700 Subject: [PATCH 16/23] Report file size of cached packages. --- install_and_cache_pkgs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index e6533a1..e4fef7f 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -61,8 +61,8 @@ for package in ${normalized_packages}; do while IFS= read -r f; do if test -f $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows done | - xargs tar -czf "${cache_filepath}" -C / - echo "done." + xargs tar -czf "${cache_filepath}" -C / + echo "done (compressed size $(du -k "${cache_filepath}" | cut -f1))." fi # Comma delimited name:ver pairs in the all packages manifest. From f6ea1022c2cb59d180889e04fca5de1516b711ee Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 07:13:58 -0700 Subject: [PATCH 17/23] Add log timings for perf debugging. --- install_and_cache_pkgs.sh | 28 ++++++++++++++-------------- lib.sh | 10 ++++++++++ pre_cache_action.sh | 28 ++++++++++++++-------------- restore_pkgs.sh | 16 ++++++++-------- 4 files changed, 46 insertions(+), 36 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index e4fef7f..f8e9ce0 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -17,22 +17,22 @@ input_packages="${@:2}" normalized_packages="$(normalize_package_list "${input_packages}")" package_count=$(wc -w <<< "${normalized_packages}") -echo "Clean installing and caching ${package_count} package(s)." -echo "Package list:" +log "Clean installing and caching ${package_count} package(s)." +log "Package list:" for package in ${normalized_packages}; do - echo "- ${package}" + log "- ${package}" done -echo -n "Updating APT package list..." +log -n "Updating APT package list..." sudo apt-get update > /dev/null -echo "done." +"done." # Strictly contains the requested packages. manifest_main="" # Contains all packages including dependencies. manifest_all="" -echo "Clean installing and caching ${package_count} packages..." +log "Clean installing and caching ${package_count} packages..." for package in ${normalized_packages}; do read package_name package_ver < <(get_package_name_ver "${package}") @@ -42,10 +42,10 @@ for package in ${normalized_packages}; do all_packages="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')" dep_packages="$(echo ${dep_packages} | grep -v "${package_name}" | tr '\n' ,)" - echo "- ${package_name}" - echo " * Version: ${package_ver}" - echo " * Dependencies: ${dep_packages:0:-1}" - echo -n " * Installing..." + log "- ${package_name}" + log " * Version: ${package_ver}" + log " * Dependencies: ${dep_packages:0:-1}" + log -n " * Installing..." # Zero interaction while installing or upgrading the system via apt. sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null echo "done." @@ -55,7 +55,7 @@ for package in ${normalized_packages}; do if test ! -f "${cache_filepath}"; then read cache_package_name cache_package_ver < <(get_package_name_ver "${cache_package}") - echo -n " * Caching ${cache_package_name} to ${cache_filepath}..." + log -n " * Caching ${cache_package_name} to ${cache_filepath}..." # Pipe all package files (no folders) to Tar. dpkg -L "${cache_package_name}" | while IFS= read -r f; do @@ -69,16 +69,16 @@ for package in ${normalized_packages}; do manifest_all="${manifest_all}${cache_package_name}:${cache_package_ver}," done done -echo "done." +log "done." manifest_all_filepath="${cache_dir}/manifest_all.log" -echo -n "Writing all packages manifest to ${manifest_all_filepath}..." +log -n "Writing all packages manifest to ${manifest_all_filepath}..." # Remove trailing comma and write to manifest_all file. echo "${manifest_all:0:-1}" > "${manifest_all_filepath}" echo "done." manifest_main_filepath="${cache_dir}/manifest_main.log" -echo -n "Writing main requested packages manifest to ${manifest_main_filepath}..." +log -n "Writing main requested packages manifest to ${manifest_main_filepath}..." # Remove trailing comma and write to manifest_main file. echo "${manifest_main:0:-1}" > "${manifest_main_filepath}" echo "done." \ No newline at end of file diff --git a/lib.sh b/lib.sh index 0c9ddd1..2c216be 100755 --- a/lib.sh +++ b/lib.sh @@ -17,3 +17,13 @@ function get_package_name_ver { fi echo "${name}" "${ver}" } + +function log { + timestamp="$(echo -n "$(date +%H:%M:%S)")" + line=""${timestamp}" "$(echo ${@})"" + if [[ "${1}" == "-n" ]]; then + echo -n "${line}" + else + echo "${line}" + fi +} \ No newline at end of file diff --git a/pre_cache_action.sh b/pre_cache_action.sh index ec33dae..19f6594 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -19,27 +19,27 @@ packages="$(normalize_package_list "${input_packages}")" # Create cache directory so artifacts can be saved. mkdir -p ${cache_dir} -echo -n "Validating action arguments (version='${version}', packages='${packages}')..."; +log -n "Validating action arguments (version='${version}', packages='${packages}')..."; if grep -q " " <<< "${version}"; then - echo "aborted." - echo "Version value '${version}' cannot contain spaces." >&2 + log "aborted." + log "Version value '${version}' cannot contain spaces." >&2 exit 1 fi # Is length of string zero? if test -z "${packages}"; then - echo "aborted." - echo "Packages argument cannot be empty." >&2 + log "aborted." + log "Packages argument cannot be empty." >&2 exit 2 fi -echo "done." +log "done." versioned_packages="" -echo -n "Verifying packages..." +log -n "Verifying packages..." for package in ${packages}; do if test ! "$(apt show "${package}")"; then echo "aborted." - echo "Package '${package}' not found." >&2 + log "Package '${package}' not found." >&2 exit 3 fi read package_name package_ver < <(get_package_name_ver "${package}") @@ -50,20 +50,20 @@ echo "done." # Abort on any failure at this point. set -e -echo "Creating cache key..." +log "Creating cache key..." # TODO Can we prove this will happen again? normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")" -echo "- Normalized package list is '${normalized_versioned_packages}'." +log "- Normalized package list is '${normalized_versioned_packages}'." value="$(echo "${normalized_versioned_packages} @ ${version}")" -echo "- Value to hash is '${value}'." +log "- Value to hash is '${value}'." key="$(echo "${value}" | md5sum | /bin/cut -f1 -d' ')" -echo "- Value hashed as '${key}'." +log "- Value hashed as '${key}'." -echo "done." +log "done." key_filepath="${cache_dir}/cache_key.md5" echo ${key} > ${key_filepath} -echo "Hash value written to ${key_filepath}" +log "Hash value written to ${key_filepath}" diff --git a/restore_pkgs.sh b/restore_pkgs.sh index 655490b..647a2aa 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -11,24 +11,24 @@ cache_dir="${1}" cache_restore_root="${2}" cache_filepaths="$(ls -1 "${cache_dir}" | sort)" -echo "Found $(echo ${cache_filepaths} | wc -w) files in the cache." +log "Found $(echo ${cache_filepaths} | wc -w) files in the cache." for cache_filepath in ${cache_filepaths}; do - echo "- "$(basename ${cache_filepath})"" + log "- "$(basename ${cache_filepath})"" done -echo "Reading from main requested packages manifest..." +log "Reading from main requested packages manifest..." for logline in $(cat "${cache_dir}/manifest_main.log" | tr ',' '\n' ); do - echo "- $(echo "${logline}" | tr ':' ' ')" + log "- $(echo "${logline}" | tr ':' ' ')" done -echo "done." +log "done." # Only search for archived results. Manifest and cache key also live here. cache_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort) cache_pkg_filecount=$(echo ${cache_pkg_filepaths} | wc -w) -echo "Restoring ${cache_pkg_filecount} packages from cache..." +log "Restoring ${cache_pkg_filecount} packages from cache..." for cache_pkg_filepath in ${cache_pkg_filepaths}; do - echo -n "- $(basename "${cache_pkg_filepath}") restoring..." + log -n "- $(basename "${cache_pkg_filepath}") restoring..." sudo tar -xf "${cache_pkg_filepath}" -C "${cache_restore_root}" > /dev/null echo "done." done -echo "done." +log "done." From 7274c01428dc0e8a01a993528db00af76a217b6c Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 08:24:18 -0700 Subject: [PATCH 18/23] Fix minor bug. --- install_and_cache_pkgs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index f8e9ce0..e22cabd 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -25,7 +25,7 @@ done log -n "Updating APT package list..." sudo apt-get update > /dev/null -"done." +echo "done." # Strictly contains the requested packages. manifest_main="" From 60c15ce2f095e46d08d546a6c8d22274083a2148 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 08:28:26 -0700 Subject: [PATCH 19/23] Include library in all scripts. --- post_cache_action.sh | 4 ++++ restore_pkgs.sh | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/post_cache_action.sh b/post_cache_action.sh index 8440aa3..144fe76 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -3,6 +3,10 @@ # Fail on any error. set -e +# Include library. +script_dir="$(dirname -- "$(realpath -- "${0}")")" +source "${script_dir}/lib.sh" + # Directory that holds the cached packages. cache_dir="${1}" diff --git a/restore_pkgs.sh b/restore_pkgs.sh index 647a2aa..d056630 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -3,6 +3,10 @@ # Fail on any error. set -e +# Include library. +script_dir="$(dirname -- "$(realpath -- "${0}")")" +source "${script_dir}/lib.sh" + # Directory that holds the cached packages. cache_dir="${1}" From 768417e4ec4577eba696a089c7d63892db492d16 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 30 Jun 2022 08:41:19 -0700 Subject: [PATCH 20/23] Fix dependency listing. --- install_and_cache_pkgs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index e22cabd..206acb0 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -40,7 +40,7 @@ for package in ${normalized_packages}; do manifest_main="${manifest_main}${package_name}:${package_ver}," all_packages="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')" - dep_packages="$(echo ${dep_packages} | grep -v "${package_name}" | tr '\n' ,)" + dep_packages="$(echo ${all_packages} | grep -v "${package_name}" | tr '\n' ,)" log "- ${package_name}" log " * Version: ${package_ver}" From 39faaf9bee7f289b21cf9f43ea4a6750c82619cc Mon Sep 17 00:00:00 2001 From: Andrew Walsh Date: Thu, 14 Jul 2022 21:19:19 -0700 Subject: [PATCH 21/23] Update installation to consider symlinks as well. Addresses problem raised in #25 --- install_and_cache_pkgs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 206acb0..4411e24 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -59,7 +59,7 @@ for package in ${normalized_packages}; do # Pipe all package files (no folders) to Tar. dpkg -L "${cache_package_name}" | while IFS= read -r f; do - if test -f $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows + if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows done | xargs tar -czf "${cache_filepath}" -C / echo "done (compressed size $(du -k "${cache_filepath}" | cut -f1))." @@ -81,4 +81,4 @@ manifest_main_filepath="${cache_dir}/manifest_main.log" log -n "Writing main requested packages manifest to ${manifest_main_filepath}..." # Remove trailing comma and write to manifest_main file. echo "${manifest_main:0:-1}" > "${manifest_main_filepath}" -echo "done." \ No newline at end of file +echo "done." From 63f7fccab9c795693ca5a4bfeab0da91271d9478 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 14 Jul 2022 21:44:35 -0700 Subject: [PATCH 22/23] Simplify logging calls. --- install_and_cache_pkgs.sh | 21 ++++++++++++--------- lib.sh | 16 ++++------------ pre_cache_action.sh | 2 +- restore_pkgs.sh | 4 ++-- 4 files changed, 19 insertions(+), 24 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 206acb0..23580e0 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -23,7 +23,7 @@ for package in ${normalized_packages}; do log "- ${package}" done -log -n "Updating APT package list..." +log "Updating APT package list..." sudo apt-get update > /dev/null echo "done." @@ -41,11 +41,14 @@ for package in ${normalized_packages}; do all_packages="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')" dep_packages="$(echo ${all_packages} | grep -v "${package_name}" | tr '\n' ,)" + if "${dep_packages}" == ","; then + dep_packages="none"; + fi log "- ${package_name}" log " * Version: ${package_ver}" - log " * Dependencies: ${dep_packages:0:-1}" - log -n " * Installing..." + log " * Dependencies: ${dep_packages}" + log " * Installing..." # Zero interaction while installing or upgrading the system via apt. sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null echo "done." @@ -55,14 +58,14 @@ for package in ${normalized_packages}; do if test ! -f "${cache_filepath}"; then read cache_package_name cache_package_ver < <(get_package_name_ver "${cache_package}") - log -n " * Caching ${cache_package_name} to ${cache_filepath}..." + log " * Caching ${cache_package_name} to ${cache_filepath}..." # Pipe all package files (no folders) to Tar. dpkg -L "${cache_package_name}" | while IFS= read -r f; do if test -f $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows done | xargs tar -czf "${cache_filepath}" -C / - echo "done (compressed size $(du -k "${cache_filepath}" | cut -f1))." + log "done (compressed size $(du -k "${cache_filepath}" | cut -f1))." fi # Comma delimited name:ver pairs in the all packages manifest. @@ -72,13 +75,13 @@ done log "done." manifest_all_filepath="${cache_dir}/manifest_all.log" -log -n "Writing all packages manifest to ${manifest_all_filepath}..." +log "Writing all packages manifest to ${manifest_all_filepath}..." # Remove trailing comma and write to manifest_all file. echo "${manifest_all:0:-1}" > "${manifest_all_filepath}" -echo "done." +log "done." manifest_main_filepath="${cache_dir}/manifest_main.log" -log -n "Writing main requested packages manifest to ${manifest_main_filepath}..." +log "Writing main requested packages manifest to ${manifest_main_filepath}..." # Remove trailing comma and write to manifest_main file. echo "${manifest_main:0:-1}" > "${manifest_main_filepath}" -echo "done." \ No newline at end of file +log "done." \ No newline at end of file diff --git a/lib.sh b/lib.sh index 2c216be..0a9c580 100755 --- a/lib.sh +++ b/lib.sh @@ -2,10 +2,10 @@ # Sort these packages by name and split on commas. function normalize_package_list { - stripped=$(echo "${1}" | sed 's/,//g') + local stripped=$(echo "${1}" | sed 's/,//g') # Remove extraneous spaces at the middle, beginning, and end. - trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" - echo "$(echo "${trimmed}" | sort)" + local trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" + echo "${trimmed}" | sort } # Split fully qualified package into name and version @@ -18,12 +18,4 @@ function get_package_name_ver { echo "${name}" "${ver}" } -function log { - timestamp="$(echo -n "$(date +%H:%M:%S)")" - line=""${timestamp}" "$(echo ${@})"" - if [[ "${1}" == "-n" ]]; then - echo -n "${line}" - else - echo "${line}" - fi -} \ No newline at end of file +function log { echo "$(date +%H:%M:%S)" "${@}"; } \ No newline at end of file diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 19f6594..7921adb 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -56,7 +56,7 @@ log "Creating cache key..." normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")" log "- Normalized package list is '${normalized_versioned_packages}'." -value="$(echo "${normalized_versioned_packages} @ ${version}")" +value="${normalized_versioned_packages} @ ${version}" log "- Value to hash is '${value}'." key="$(echo "${value}" | md5sum | /bin/cut -f1 -d' ')" diff --git a/restore_pkgs.sh b/restore_pkgs.sh index d056630..ec6c475 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -31,8 +31,8 @@ cache_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort) cache_pkg_filecount=$(echo ${cache_pkg_filepaths} | wc -w) log "Restoring ${cache_pkg_filecount} packages from cache..." for cache_pkg_filepath in ${cache_pkg_filepaths}; do - log -n "- $(basename "${cache_pkg_filepath}") restoring..." + log "- $(basename "${cache_pkg_filepath}") restoring..." sudo tar -xf "${cache_pkg_filepath}" -C "${cache_restore_root}" > /dev/null - echo "done." + log "done." done log "done." From fa7091e9a8c98a3c44faf1a7df8339f90cdf291b Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Thu, 14 Jul 2022 21:57:20 -0700 Subject: [PATCH 23/23] Fix sorting when normalizing package list. --- lib.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib.sh b/lib.sh index 0a9c580..f3757c9 100755 --- a/lib.sh +++ b/lib.sh @@ -4,8 +4,9 @@ function normalize_package_list { local stripped=$(echo "${1}" | sed 's/,//g') # Remove extraneous spaces at the middle, beginning, and end. - local trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" - echo "${trimmed}" | sort + local trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" + local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')" + echo "${sorted}" } # Split fully qualified package into name and version