catmeow/cache-apt-pkgs-action
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'staging'

Browse source
This commit is contained in:
awalsh128 2022-07-15 15:28:17 -07:00
commit 1d2f5e1b23
7 changed files with 166 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -24,8 +24,8 @@ Create a workflow `.yml` file in your repositories `.github/workflows` directory
### Outputs ### Outputs
* `cache-hit` - A boolean value to indicate a cache was found for the packages requested. * `cache-hit` - A boolean value to indicate a cache was found for the packages requested.
* `package-version-list` - The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. \<package1>:<version1\>,\<package2>:\<version2>,...). * `package-version-list` - The main requested packages and versions that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. \<package1>:<version1\>,\<package2>:\<version2>,...).
* `all-package-version-list` - All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. \<package1>:<version1\>,\<package2>:\<version2>,...).
### Cache scopes ### Cache scopes

8
action.yml
View file

@ -26,7 +26,10 @@ outputs:
# Need to output true and false instead of true and nothing. # Need to output true and false instead of true and nothing.
value: ${{ steps.load-cache.outputs.cache-hit || false }} value: ${{ steps.load-cache.outputs.cache-hit || false }}
package-version-list: package-version-list:
description: 'The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. <package>:<version,<package>:<version>).' description: 'The main requested packages and versions that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. <package>:<version,<package>:<version>).'
value: ${{ steps.post-cache.outputs.package-version-list }}
all-package-version-list:
description: 'All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. <package>:<version,<package>:<version>).'
value: ${{ steps.post-cache.outputs.package-version-list }} value: ${{ steps.post-cache.outputs.package-version-list }}
runs: runs:
@ -54,5 +57,6 @@ runs:
/ \ / \
"${{ steps.load-cache.outputs.cache-hit }}" \ "${{ steps.load-cache.outputs.cache-hit }}" \
${{ inputs.packages }} ${{ inputs.packages }}
echo "::set-output name=package-version-list::$(cat ~/cache-apt-pkgs/manifest.log)" echo "::set-output name=package-version-list::$(cat ~/cache-apt-pkgs/manifest_main.log)"
echo "::set-output name=all-package-version-list::$(cat ~/cache-apt-pkgs/manifest_all.log)"
shell: bash shell: bash

87
install_and_cache_pkgs.sh
View file

@ -7,45 +7,78 @@ set -e
cache_dir=$1 cache_dir=$1
# List of the packages to use. # List of the packages to use.
packages="${@:2}" input_packages="${@:2}"
package_count=$(echo $packages | wc -w) # Trim commas, excess spaces, and sort.
echo "Clean installing and caching $package_count package(s)." normalized_packages="$(normalize_package_list "${input_packages}")"
echo "Package list:"
for package in $packages; do package_count=$(wc -w <<< "${normalized_packages}")
echo "- $package" log "Clean installing and caching ${package_count} package(s)."
log "Package list:"
for package in ${normalized_packages}; do
log "- ${package}"
done done
echo -n "Updating APT package list..." log "Updating APT package list..."
sudo apt-get update > /dev/null sudo apt-get update > /dev/null
echo "done." echo "done."
echo "Clean installing and caching $(echo $packages | wc -w) packages..." # Strictly contains the requested packages.
for package in $packages; do manifest_main=""
cache_filepath=$cache_dir/$package.tar.gz # Contains all packages including dependencies.
manifest_all=""
echo "- $package" log "Clean installing and caching ${package_count} packages..."
echo -n " Installing..." for package in ${normalized_packages}; do
sudo apt-get --yes install $package > /dev/null read package_name package_ver < <(get_package_name_ver "${package}")
# Comma delimited name:ver pairs in the main requested packages manifest.
manifest_main="${manifest_main}${package_name}:${package_ver},"
all_packages="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')"
dep_packages="$(echo ${all_packages} | grep -v "${package_name}" | tr '\n' ,)"
if "${dep_packages}" == ","; then
dep_packages="none";
fi
log "- ${package_name}"
log " * Version: ${package_ver}"
log " * Dependencies: ${dep_packages}"
log " * Installing..."
# Zero interaction while installing or upgrading the system via apt.
sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null
echo "done." echo "done."
echo -n " Caching to $cache_filepath..." for cache_package in ${all_packages}; do
cache_filepath="${cache_dir}/${cache_package}.tar.gz"
if test ! -f "${cache_filepath}"; then
read cache_package_name cache_package_ver < <(get_package_name_ver "${cache_package}")
log " * Caching ${cache_package_name} to ${cache_filepath}..."
# Pipe all package files (no folders) to Tar. # Pipe all package files (no folders) to Tar.
dpkg -L $package | dpkg -L "${cache_package_name}" |
while IFS= read -r f; do while IFS= read -r f; do
if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows
done | done |
xargs tar -czf $cache_filepath -C / xargs tar -czf "${cache_filepath}" -C /
echo "done." log "done (compressed size $(du -k "${cache_filepath}" | cut -f1))."
done fi
echo "done."
manifest_filepath="$cache_dir/manifest.log" # Comma delimited name:ver pairs in the all packages manifest.
echo -n "Writing package manifest to $manifest_filepath..." manifest_all="${manifest_all}${cache_package_name}:${cache_package_ver},"
manifest=
for package in $packages; do
manifest=$manifest$package:$(dpkg -s $package | grep Version | awk '{print $2}'),
done done
# Remove trailing comma. done
echo ${manifest:0:-1} > $manifest_filepath log "done."
echo "done."
manifest_all_filepath="${cache_dir}/manifest_all.log"
log "Writing all packages manifest to ${manifest_all_filepath}..."
# Remove trailing comma and write to manifest_all file.
echo "${manifest_all:0:-1}" > "${manifest_all_filepath}"
log "done."
manifest_main_filepath="${cache_dir}/manifest_main.log"
log "Writing main requested packages manifest to ${manifest_main_filepath}..."
# Remove trailing comma and write to manifest_main file.
echo "${manifest_main:0:-1}" > "${manifest_main_filepath}"
log "done."

22
lib.sh Executable file
View file

@ -0,0 +1,22 @@
#!/bin/bash
# Sort these packages by name and split on commas.
function normalize_package_list {
local stripped=$(echo "${1}" | sed 's/,//g')
# Remove extraneous spaces at the middle, beginning, and end.
local trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')"
local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')"
echo "${sorted}"
}
# Split fully qualified package into name and version
function get_package_name_ver {
IFS=\= read name ver <<< "${1}"
# If version not found in the fully qualified package value.
if test -z "${ver}"; then
ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')"
fi
echo "${name}" "${ver}"
}
function log { echo "$(date +%H:%M:%S)" "${@}"; }

16
post_cache_action.sh
View file

@ -3,24 +3,28 @@
# Fail on any error. # Fail on any error.
set -e set -e
# Include library.
script_dir="$(dirname -- "$(realpath -- "${0}")")"
source "${script_dir}/lib.sh"
# Directory that holds the cached packages. # Directory that holds the cached packages.
cache_dir=$1 cache_dir="${1}"
# Root directory to untar the cached packages to. # Root directory to untar the cached packages to.
# Typically filesystem root '/' but can be changed for testing. # Typically filesystem root '/' but can be changed for testing.
cache_restore_root=$2 cache_restore_root="${2}"
# Indicates that the cache was found. # Indicates that the cache was found.
cache_hit=$3 cache_hit="${3}"
# List of the packages to use. # List of the packages to use.
packages="${@:4}" packages="${@:4}"
script_dir=$(dirname $0) script_dir="$(dirname -- "$(realpath -- "${0}")")"
if [ "$cache_hit" == true ]; then if [ "$cache_hit" == true ]; then
$script_dir/restore_pkgs.sh ~/cache-apt-pkgs $cache_restore_root ${script_dir}/restore_pkgs.sh ~/cache-apt-pkgs "${cache_restore_root}"
else else
$script_dir/install_and_cache_pkgs.sh ~/cache-apt-pkgs $packages ${script_dir}/install_and_cache_pkgs.sh ~/cache-apt-pkgs ${packages}
fi fi
echo "" echo ""

71
pre_cache_action.sh
View file

@ -1,60 +1,69 @@
#!/bin/bash #!/bin/bash
# Include library.
script_dir="$(dirname -- "$(realpath -- "${0}")")"
source "${script_dir}/lib.sh"
# Directory that holds the cached packages. # Directory that holds the cached packages.
cache_dir=$1 cache_dir="${1}"
# Version of the cache to create or load. # Version of the cache to create or load.
version=$2 version="${2}"
# List of the packages to use. # List of the packages to use.
packages=${@:3} input_packages="${@:3}"
# Trim commas, excess spaces, and sort.
packages="$(normalize_package_list "${input_packages}")"
# Create cache directory so artifacts can be saved. # Create cache directory so artifacts can be saved.
mkdir -p $cache_dir mkdir -p ${cache_dir}
echo -n "Validating action arguments (version='$version', packages='$packages')..."; log -n "Validating action arguments (version='${version}', packages='${packages}')...";
echo $version | grep -o " " > /dev/null if grep -q " " <<< "${version}"; then
if [ $? -eq 0 ]; then log "aborted."
echo "aborted." log "Version value '${version}' cannot contain spaces." >&2
echo "Version value '$version' cannot contain spaces." >&2
exit 1 exit 1
fi fi
if [ "$packages" == "" ]; then
echo "aborted." # Is length of string zero?
echo "Packages argument cannot be empty." >&2 if test -z "${packages}"; then
log "aborted."
log "Packages argument cannot be empty." >&2
exit 2 exit 2
fi fi
echo "done." log "done."
echo -n "Verifying packages..." versioned_packages=""
for package in $packages; do log -n "Verifying packages..."
escaped=$(echo $package | sed 's/+/\\+/g') for package in ${packages}; do
apt-cache search ^$escaped$ | grep $package > /dev/null if test ! "$(apt show "${package}")"; then
if [ $? -ne 0 ]; then
echo "aborted." echo "aborted."
echo "Package '$package' not found." >&2 log "Package '${package}' not found." >&2
exit 3 exit 3
fi fi
read package_name package_ver < <(get_package_name_ver "${package}")
versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}""
done done
echo "done." echo "done."
# Abort on any failure at this point. # Abort on any failure at this point.
set -e set -e
echo "Creating cache key..." log "Creating cache key..."
# Remove package delimiters, sort (requires newline) and then convert back to inline list. # TODO Can we prove this will happen again?
normalized_list=$(echo $packages | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")"
echo "- Normalized package list is '$normalized_list'." log "- Normalized package list is '${normalized_versioned_packages}'."
value=$(echo $normalized_list @ $version) value="${normalized_versioned_packages} @ ${version}"
echo "- Value to hash is '$value'." log "- Value to hash is '${value}'."
key=$(echo $value | md5sum | cut -f1 -d' ') key="$(echo "${value}" | md5sum | /bin/cut -f1 -d' ')"
echo "- Value hashed as '$key'." log "- Value hashed as '${key}'."
echo "done." log "done."
key_filepath="$cache_dir/cache_key.md5" key_filepath="${cache_dir}/cache_key.md5"
echo $key > $key_filepath echo ${key} > ${key_filepath}
echo "Hash value written to $key_filepath" log "Hash value written to ${key_filepath}"

40
restore_pkgs.sh
View file

@ -3,32 +3,36 @@
# Fail on any error. # Fail on any error.
set -e set -e
# Include library.
script_dir="$(dirname -- "$(realpath -- "${0}")")"
source "${script_dir}/lib.sh"
# Directory that holds the cached packages. # Directory that holds the cached packages.
cache_dir=$1 cache_dir="${1}"
# Root directory to untar the cached packages to. # Root directory to untar the cached packages to.
# Typically filesystem root '/' but can be changed for testing. # Typically filesystem root '/' but can be changed for testing.
cache_restore_root=$2 cache_restore_root="${2}"
cache_filepaths=$(ls -1 $cache_dir | sort) cache_filepaths="$(ls -1 "${cache_dir}" | sort)"
echo "Found $(echo $cache_filepaths | wc -w) files in the cache." log "Found $(echo ${cache_filepaths} | wc -w) files in the cache."
for cache_filepath in $cache_filepaths; do for cache_filepath in ${cache_filepaths}; do
echo "- $(basename $cache_filepath)" log "- "$(basename ${cache_filepath})""
done done
echo "Reading from manifest..." log "Reading from main requested packages manifest..."
for logline in $(cat $cache_dir/manifest.log | tr ',' '\n' ); do for logline in $(cat "${cache_dir}/manifest_main.log" | tr ',' '\n' ); do
echo "- $(echo $logline | tr ':' ' ')" log "- $(echo "${logline}" | tr ':' ' ')"
done done
echo "done." log "done."
# Only search for archived results. Manifest and cache key also live here. # Only search for archived results. Manifest and cache key also live here.
cache_pkg_filepaths=$(ls -1 $cache_dir/*.tar.gz | sort) cache_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort)
cache_pkg_filecount=$(echo $cache_pkg_filepaths | wc -w) cache_pkg_filecount=$(echo ${cache_pkg_filepaths} | wc -w)
echo "Restoring $cache_pkg_filecount packages from cache..." log "Restoring ${cache_pkg_filecount} packages from cache..."
for cache_pkg_filepath in $cache_pkg_filepaths; do for cache_pkg_filepath in ${cache_pkg_filepaths}; do
echo -n "- $(basename $cache_pkg_filepath) restoring..." log "- $(basename "${cache_pkg_filepath}") restoring..."
sudo tar -xf $cache_pkg_filepath -C $cache_restore_root > /dev/null sudo tar -xf "${cache_pkg_filepath}" -C "${cache_restore_root}" > /dev/null
echo "done." log "done."
done done
echo "done." log "done."