diff --git a/README.md b/README.md index 927e93a..972ce8a 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ Create a workflow `.yml` file in your repositories `.github/workflows` directory ### Outputs * `cache-hit` - A boolean value to indicate a cache was found for the packages requested. -* `package-version-list` - The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. \:,\:\,...). - +* `package-version-list` - The main requested packages and versions that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. \:,\:\,...). +* `all-package-version-list` - All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. \:,\:\,...). ### Cache scopes diff --git a/action.yml b/action.yml index 3ef2f6a..9cd2164 100644 --- a/action.yml +++ b/action.yml @@ -26,7 +26,10 @@ outputs: # Need to output true and false instead of true and nothing. value: ${{ steps.load-cache.outputs.cache-hit || false }} package-version-list: - description: 'The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. ::).' + description: 'The main requested packages and versions that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. ::).' + value: ${{ steps.post-cache.outputs.package-version-list }} + all-package-version-list: + description: 'All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. ::).' value: ${{ steps.post-cache.outputs.package-version-list }} runs: @@ -54,5 +57,6 @@ runs: / \ "${{ steps.load-cache.outputs.cache-hit }}" \ ${{ inputs.packages }} - echo "::set-output name=package-version-list::$(cat ~/cache-apt-pkgs/manifest.log)" + echo "::set-output name=package-version-list::$(cat ~/cache-apt-pkgs/manifest_main.log)" + echo "::set-output name=all-package-version-list::$(cat ~/cache-apt-pkgs/manifest_all.log)" shell: bash diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 4de889d..467f980 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -7,45 +7,78 @@ set -e cache_dir=$1 # List of the packages to use. -packages="${@:2}" +input_packages="${@:2}" -package_count=$(echo $packages | wc -w) -echo "Clean installing and caching $package_count package(s)." -echo "Package list:" -for package in $packages; do - echo "- $package" +# Trim commas, excess spaces, and sort. +normalized_packages="$(normalize_package_list "${input_packages}")" + +package_count=$(wc -w <<< "${normalized_packages}") +log "Clean installing and caching ${package_count} package(s)." +log "Package list:" +for package in ${normalized_packages}; do + log "- ${package}" done -echo -n "Updating APT package list..." +log "Updating APT package list..." sudo apt-get update > /dev/null echo "done." -echo "Clean installing and caching $(echo $packages | wc -w) packages..." -for package in $packages; do - cache_filepath=$cache_dir/$package.tar.gz +# Strictly contains the requested packages. +manifest_main="" +# Contains all packages including dependencies. +manifest_all="" - echo "- $package" - echo -n " Installing..." - sudo apt-get --yes install $package > /dev/null +log "Clean installing and caching ${package_count} packages..." +for package in ${normalized_packages}; do + read package_name package_ver < <(get_package_name_ver "${package}") + + # Comma delimited name:ver pairs in the main requested packages manifest. + manifest_main="${manifest_main}${package_name}:${package_ver}," + + all_packages="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')" + dep_packages="$(echo ${all_packages} | grep -v "${package_name}" | tr '\n' ,)" + if "${dep_packages}" == ","; then + dep_packages="none"; + fi + + log "- ${package_name}" + log " * Version: ${package_ver}" + log " * Dependencies: ${dep_packages}" + log " * Installing..." + # Zero interaction while installing or upgrading the system via apt. + sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null echo "done." - echo -n " Caching to $cache_filepath..." - # Pipe all package files (no folders) to Tar. - dpkg -L $package | - while IFS= read -r f; do - if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows - done | - xargs tar -czf $cache_filepath -C / - echo "done." + for cache_package in ${all_packages}; do + cache_filepath="${cache_dir}/${cache_package}.tar.gz" + + if test ! -f "${cache_filepath}"; then + read cache_package_name cache_package_ver < <(get_package_name_ver "${cache_package}") + log " * Caching ${cache_package_name} to ${cache_filepath}..." + # Pipe all package files (no folders) to Tar. + dpkg -L "${cache_package_name}" | + while IFS= read -r f; do + if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows + done | + xargs tar -czf "${cache_filepath}" -C / + log "done (compressed size $(du -k "${cache_filepath}" | cut -f1))." + fi + + # Comma delimited name:ver pairs in the all packages manifest. + manifest_all="${manifest_all}${cache_package_name}:${cache_package_ver}," + done done -echo "done." +log "done." + +manifest_all_filepath="${cache_dir}/manifest_all.log" +log "Writing all packages manifest to ${manifest_all_filepath}..." +# Remove trailing comma and write to manifest_all file. +echo "${manifest_all:0:-1}" > "${manifest_all_filepath}" +log "done." + +manifest_main_filepath="${cache_dir}/manifest_main.log" +log "Writing main requested packages manifest to ${manifest_main_filepath}..." +# Remove trailing comma and write to manifest_main file. +echo "${manifest_main:0:-1}" > "${manifest_main_filepath}" +log "done." -manifest_filepath="$cache_dir/manifest.log" -echo -n "Writing package manifest to $manifest_filepath..." -manifest= -for package in $packages; do - manifest=$manifest$package:$(dpkg -s $package | grep Version | awk '{print $2}'), -done -# Remove trailing comma. -echo ${manifest:0:-1} > $manifest_filepath -echo "done." diff --git a/lib.sh b/lib.sh new file mode 100755 index 0000000..f3757c9 --- /dev/null +++ b/lib.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Sort these packages by name and split on commas. +function normalize_package_list { + local stripped=$(echo "${1}" | sed 's/,//g') + # Remove extraneous spaces at the middle, beginning, and end. + local trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" + local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')" + echo "${sorted}" +} + +# Split fully qualified package into name and version +function get_package_name_ver { + IFS=\= read name ver <<< "${1}" + # If version not found in the fully qualified package value. + if test -z "${ver}"; then + ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')" + fi + echo "${name}" "${ver}" +} + +function log { echo "$(date +%H:%M:%S)" "${@}"; } \ No newline at end of file diff --git a/post_cache_action.sh b/post_cache_action.sh index 5503013..144fe76 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -3,24 +3,28 @@ # Fail on any error. set -e +# Include library. +script_dir="$(dirname -- "$(realpath -- "${0}")")" +source "${script_dir}/lib.sh" + # Directory that holds the cached packages. -cache_dir=$1 +cache_dir="${1}" # Root directory to untar the cached packages to. # Typically filesystem root '/' but can be changed for testing. -cache_restore_root=$2 +cache_restore_root="${2}" # Indicates that the cache was found. -cache_hit=$3 +cache_hit="${3}" # List of the packages to use. packages="${@:4}" -script_dir=$(dirname $0) +script_dir="$(dirname -- "$(realpath -- "${0}")")" if [ "$cache_hit" == true ]; then - $script_dir/restore_pkgs.sh ~/cache-apt-pkgs $cache_restore_root + ${script_dir}/restore_pkgs.sh ~/cache-apt-pkgs "${cache_restore_root}" else - $script_dir/install_and_cache_pkgs.sh ~/cache-apt-pkgs $packages + ${script_dir}/install_and_cache_pkgs.sh ~/cache-apt-pkgs ${packages} fi echo "" diff --git a/pre_cache_action.sh b/pre_cache_action.sh index ba09879..7921adb 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -1,60 +1,69 @@ #!/bin/bash +# Include library. +script_dir="$(dirname -- "$(realpath -- "${0}")")" +source "${script_dir}/lib.sh" + # Directory that holds the cached packages. -cache_dir=$1 +cache_dir="${1}" # Version of the cache to create or load. -version=$2 +version="${2}" # List of the packages to use. -packages=${@:3} +input_packages="${@:3}" + +# Trim commas, excess spaces, and sort. +packages="$(normalize_package_list "${input_packages}")" # Create cache directory so artifacts can be saved. -mkdir -p $cache_dir +mkdir -p ${cache_dir} -echo -n "Validating action arguments (version='$version', packages='$packages')..."; -echo $version | grep -o " " > /dev/null -if [ $? -eq 0 ]; then - echo "aborted." - echo "Version value '$version' cannot contain spaces." >&2 +log -n "Validating action arguments (version='${version}', packages='${packages}')..."; +if grep -q " " <<< "${version}"; then + log "aborted." + log "Version value '${version}' cannot contain spaces." >&2 exit 1 fi -if [ "$packages" == "" ]; then - echo "aborted." - echo "Packages argument cannot be empty." >&2 + +# Is length of string zero? +if test -z "${packages}"; then + log "aborted." + log "Packages argument cannot be empty." >&2 exit 2 fi -echo "done." +log "done." -echo -n "Verifying packages..." -for package in $packages; do - escaped=$(echo $package | sed 's/+/\\+/g') - apt-cache search ^$escaped$ | grep $package > /dev/null - if [ $? -ne 0 ]; then +versioned_packages="" +log -n "Verifying packages..." +for package in ${packages}; do + if test ! "$(apt show "${package}")"; then echo "aborted." - echo "Package '$package' not found." >&2 + log "Package '${package}' not found." >&2 exit 3 fi + read package_name package_ver < <(get_package_name_ver "${package}") + versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}"" done echo "done." # Abort on any failure at this point. set -e -echo "Creating cache key..." +log "Creating cache key..." -# Remove package delimiters, sort (requires newline) and then convert back to inline list. -normalized_list=$(echo $packages | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') -echo "- Normalized package list is '$normalized_list'." +# TODO Can we prove this will happen again? +normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")" +log "- Normalized package list is '${normalized_versioned_packages}'." -value=$(echo $normalized_list @ $version) -echo "- Value to hash is '$value'." +value="${normalized_versioned_packages} @ ${version}" +log "- Value to hash is '${value}'." -key=$(echo $value | md5sum | cut -f1 -d' ') -echo "- Value hashed as '$key'." +key="$(echo "${value}" | md5sum | /bin/cut -f1 -d' ')" +log "- Value hashed as '${key}'." -echo "done." +log "done." -key_filepath="$cache_dir/cache_key.md5" -echo $key > $key_filepath -echo "Hash value written to $key_filepath" +key_filepath="${cache_dir}/cache_key.md5" +echo ${key} > ${key_filepath} +log "Hash value written to ${key_filepath}" diff --git a/restore_pkgs.sh b/restore_pkgs.sh index ab87879..ec6c475 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -3,32 +3,36 @@ # Fail on any error. set -e +# Include library. +script_dir="$(dirname -- "$(realpath -- "${0}")")" +source "${script_dir}/lib.sh" + # Directory that holds the cached packages. -cache_dir=$1 +cache_dir="${1}" # Root directory to untar the cached packages to. # Typically filesystem root '/' but can be changed for testing. -cache_restore_root=$2 +cache_restore_root="${2}" -cache_filepaths=$(ls -1 $cache_dir | sort) -echo "Found $(echo $cache_filepaths | wc -w) files in the cache." -for cache_filepath in $cache_filepaths; do - echo "- $(basename $cache_filepath)" +cache_filepaths="$(ls -1 "${cache_dir}" | sort)" +log "Found $(echo ${cache_filepaths} | wc -w) files in the cache." +for cache_filepath in ${cache_filepaths}; do + log "- "$(basename ${cache_filepath})"" done -echo "Reading from manifest..." -for logline in $(cat $cache_dir/manifest.log | tr ',' '\n' ); do - echo "- $(echo $logline | tr ':' ' ')" +log "Reading from main requested packages manifest..." +for logline in $(cat "${cache_dir}/manifest_main.log" | tr ',' '\n' ); do + log "- $(echo "${logline}" | tr ':' ' ')" done -echo "done." +log "done." # Only search for archived results. Manifest and cache key also live here. -cache_pkg_filepaths=$(ls -1 $cache_dir/*.tar.gz | sort) -cache_pkg_filecount=$(echo $cache_pkg_filepaths | wc -w) -echo "Restoring $cache_pkg_filecount packages from cache..." -for cache_pkg_filepath in $cache_pkg_filepaths; do - echo -n "- $(basename $cache_pkg_filepath) restoring..." - sudo tar -xf $cache_pkg_filepath -C $cache_restore_root > /dev/null - echo "done." +cache_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort) +cache_pkg_filecount=$(echo ${cache_pkg_filepaths} | wc -w) +log "Restoring ${cache_pkg_filecount} packages from cache..." +for cache_pkg_filepath in ${cache_pkg_filepaths}; do + log "- $(basename "${cache_pkg_filepath}") restoring..." + sudo tar -xf "${cache_pkg_filepath}" -C "${cache_restore_root}" > /dev/null + log "done." done -echo "done." +log "done."