From e68df30bd6bff9a45d3de3069b72247c443d7e3c Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Sat, 26 Mar 2022 12:42:40 -0700 Subject: [PATCH 1/2] Experimental version. Cleaned up syntax and created common library functions. --- install_and_cache_pkgs.sh | 41 ++++++++++++++++++----------------- lib.sh | 23 ++++++++++++++++++++ pre_cache_action.sh | 45 ++++++++++++++++++--------------------- restore_pkgs.sh | 2 +- 4 files changed, 67 insertions(+), 44 deletions(-) create mode 100644 lib.sh diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index ca6c838..28b3858 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -3,22 +3,23 @@ # Fail on any error. set -e +# Include library. +script_dir="$(dirname -- "$(realpath -- "${0}")")" +source "${script_dir}/lib" + # Directory that holds the cached packages. cache_dir="${1}" # List of the packages to use. -packages="${@:2}" +input_packages="${@:2}" -# Sort these packages by name and split on commas. -packages=$(echo "${packages}" | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') - -# Remove extraneous spaces -packages="$(echo "${packages}" | sed 's/\s\+/ /g;s/^\s\+//g;s/\s\+$//g')" +# Trim commas, excess spaces, and sort. +packages="$(normalize_package_list "${input_packages}")" package_count=$(echo "${packages}" | wc -w) echo "Clean installing and caching ${package_count} package(s)." echo "Package list:" -for package in ${packages}; do +for package in "${packages}"; do echo "- ${package}" done @@ -28,21 +29,23 @@ echo "done." manifest= echo "Clean installing and caching ${package_count} packages..." -for package in ${packages}; do - echo "- ${package}" - echo -n " Installing..." +for package in "${packages}"; do + get_package_name_ver "${package}" # -> package_name, package_ver + package_deps="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')" - # Gather a list of all packages apt installs to make this package work - required_packages=$(apt-get install --dry-run --yes "${package}" | grep -Po "(?<=Inst )[^\s]+" || echo "${package}") - echo "Package ${package} installs the following packages: ${required_packages//$'\n'/, }" + echo "- ${package_name}" + echo " * Version: ${package_ver}" + echo " * Dependencies: ${package_deps}" + echo -n " * Installing..." + # Zero interaction while installing or upgrading the system via apt. sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null - echo "done." - for cache_package in ${required_packages}; do + for cache_package in "${package_deps}"; do cache_filepath="${cache_dir}/${cache_package}.tar.gz" - if [ ! -f "${cache_filepath}" ]; then - package_name="$(echo "${cache_package}" | cut -d"=" -f1)" + + if test ! -f "${cache_filepath}"; then + get_package_name_ver "${cache_package}" # -> package_name, package_ver echo -n " Caching ${package_name} to ${cache_filepath}..." # Pipe all package files (no folders) to Tar. dpkg -L "${package_name}" | @@ -60,6 +63,6 @@ echo "done." manifest_filepath="${cache_dir}/manifest.log" echo -n "Writing package manifest to ${manifest_filepath}..." -# Remove trailing comma. -echo ${manifest:0:-1} > ${manifest_filepath} +# Remove trailing comma and write to manifest file. +echo "${manifest:0:-1}" > "${manifest_filepath}" echo "done." \ No newline at end of file diff --git a/lib.sh b/lib.sh new file mode 100644 index 0000000..3802483 --- /dev/null +++ b/lib.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Sort these packages by name and split on commas. +function normalize_package_list { + stripped="$(echo \"${0}\" | sed 's/,//g')" + # Remove extraneous spaces at the middle, beginning, and end. + trimmed="$(echo \"${stripped}\" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" + echo "$(\"${trimmed}\" | sort)" +} + +# Split fully qualified package into name and version +function get_package_name_ver { + IFS=\= read name ver <<< "${0}" + # If version not found in the fully qualified package value. + if test -z "${ver}"; then + ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')" + fi + echo 'package_name="${name}"; package_ver="${ver}"' +} + +function blah { + > /dev/null 2>&1 +} \ No newline at end of file diff --git a/pre_cache_action.sh b/pre_cache_action.sh index d9dd664..d1899f5 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -1,5 +1,9 @@ #!/bin/bash +# Include library. +script_dir="$(dirname -- "$(realpath -- "${0}")")" +source "${script_dir}/lib" + # Directory that holds the cached packages. cache_dir="${1}" @@ -7,22 +11,23 @@ cache_dir="${1}" cache_version="${2}" # List of the packages to use. -packages="${@:3}" +input_packages="${@:3}" -# Sort these packages by name and split on commas. -packages=$(echo "${packages}" | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') +# Trim commas, excess spaces, and sort. +packages="$(normalize_package_list "${input_packages}")" # Create cache directory so artifacts can be saved. mkdir -p "${cache_dir}" echo -n "Validating action arguments (version='${cache_version}', packages='${packages}')..."; -if echo "${cache_version}" | grep -q " " > /dev/null; then +if grep -q " " <<< "${cache_version}"; then echo "aborted." echo "Version value '${cache_version}' cannot contain spaces." >&2 exit 1 fi -if [ "${packages}" == "" ]; then +# Is length of string zero? +if test -z "${packages}"; then echo "aborted." echo "Packages argument cannot be empty." >&2 exit 2 @@ -33,24 +38,16 @@ echo -n "Updating APT package list..." sudo apt-get update > /dev/null echo "done." +versioned_packages="" echo -n "Verifying packages..." -for package in ${packages}; do - if echo "${package}" | grep -q "="; then - package_name=$(echo "${package}" | cut -d "=" -f1) - package_ver=$(echo "${package}" | cut -d "=" -f2) - else - package_name="${package}" - fi - apt_show=$(apt show "${package}") - if echo ${apt_show} | grep -qi "No packages found" > /dev/null; then +for package in ${packages}; do + if test ! "$(apt show "${package}")"; then echo "aborted." echo "Package '${package}' not found." >&2 exit 3 fi - if [ -z "${package_ver}" ]; then - package_ver=$(echo "${apt_show}" | grep -Po "(?<=Version: )[^\s]+") - fi - package_list="${package_list} ${package_name}=${package_ver}" + get_package_name_ver "${package}" # -> package_name, package_ver + versioned_packages="${versioned_packages} ${package_name}=${package_ver}" done echo "done." @@ -59,15 +56,15 @@ set -e echo "Creating cache key..." -# Remove extraneous spaces -package_list="$(echo "${package_list}" | sed 's/\s\+/ /g;s/^\s\+//g;s/\s\+$//g')" -echo "- Normalized package list is '$package_list'." +# TODO Can we prove this will happen again? +normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")" +echo "- Normalized package list is '${normalized_versioned_packages}'." -value=$(echo "${package_list} @ ${cache_version}") +value="$(echo "${normalized_versioned_packages} @ ${cache_version}")" echo "- Value to hash is '${value}'." -key=$(echo "${value}" | md5sum | /bin/cut -f1 -d' ') -echo "- Value hashed as '$key'." +key="$(echo "${value}" | md5sum | /bin/cut -f1 -d' ')" +echo "- Value hashed as '${key}'." echo "done." diff --git a/restore_pkgs.sh b/restore_pkgs.sh index 4fdbcc6..87ee131 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -17,7 +17,7 @@ for cache_filepath in ${cache_filepaths}; do done echo "Reading from manifest..." -for logline in $(cat "${cache_dir}/manifest.log" | tr ',' '\n' ); do +for logline in "$(cat "${cache_dir}/manifest.log" | tr ',' '\n' )"; do echo "- $(echo "${logline}" | tr ':' ' ')" done echo "done." From 9568dc9783b4f9e973b956e618f5895fbda6b199 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Sat, 26 Mar 2022 12:48:16 -0700 Subject: [PATCH 2/2] Change include command. --- install_and_cache_pkgs.sh | 2 +- pre_cache_action.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 28b3858..337628f 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -5,7 +5,7 @@ set -e # Include library. script_dir="$(dirname -- "$(realpath -- "${0}")")" -source "${script_dir}/lib" +source "${script_dir}/lib.sh" # Directory that holds the cached packages. cache_dir="${1}" diff --git a/pre_cache_action.sh b/pre_cache_action.sh index d1899f5..8c8f992 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -2,7 +2,7 @@ # Include library. script_dir="$(dirname -- "$(realpath -- "${0}")")" -source "${script_dir}/lib" +source "${script_dir}/lib.sh" # Directory that holds the cached packages. cache_dir="${1}"