Merge branch 'staging'
This commit is contained in:
awalsh128
commit
1d2f5e1b23
7 changed files with 166 additions and 90 deletions
|
|
@ -24,8 +24,8 @@ Create a workflow `.yml` file in your repositories `.github/workflows` directory
|
|||
### Outputs
|
||||
|
||||
* `cache-hit` - A boolean value to indicate a cache was found for the packages requested.
|
||||
* `package-version-list` - The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. \<package1>:<version1\>,\<package2>:\<version2>,...).
|
||||
|
||||
* `package-version-list` - The main requested packages and versions that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. \<package1>:<version1\>,\<package2>:\<version2>,...).
|
||||
* `all-package-version-list` - All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. \<package1>:<version1\>,\<package2>:\<version2>,...).
|
||||
|
||||
### Cache scopes
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,10 @@ outputs:
|
|||
# Need to output true and false instead of true and nothing.
|
||||
value: ${{ steps.load-cache.outputs.cache-hit || false }}
|
||||
package-version-list:
|
||||
description: 'The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. <package>:<version,<package>:<version>).'
|
||||
description: 'The main requested packages and versions that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. <package>:<version,<package>:<version>).'
|
||||
value: ${{ steps.post-cache.outputs.package-version-list }}
|
||||
all-package-version-list:
|
||||
description: 'All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. <package>:<version,<package>:<version>).'
|
||||
value: ${{ steps.post-cache.outputs.package-version-list }}
|
||||
|
||||
runs:
|
||||
|
|
@ -54,5 +57,6 @@ runs:
|
|||
/ \
|
||||
"${{ steps.load-cache.outputs.cache-hit }}" \
|
||||
${{ inputs.packages }}
|
||||
echo "::set-output name=package-version-list::$(cat ~/cache-apt-pkgs/manifest.log)"
|
||||
echo "::set-output name=package-version-list::$(cat ~/cache-apt-pkgs/manifest_main.log)"
|
||||
echo "::set-output name=all-package-version-list::$(cat ~/cache-apt-pkgs/manifest_all.log)"
|
||||
shell: bash
|
||||
|
|
|
|||
|
|
@ -7,45 +7,78 @@ set -e
|
|||
cache_dir=$1
|
||||
|
||||
# List of the packages to use.
|
||||
packages="${@:2}"
|
||||
input_packages="${@:2}"
|
||||
|
||||
package_count=$(echo $packages | wc -w)
|
||||
echo "Clean installing and caching $package_count package(s)."
|
||||
echo "Package list:"
|
||||
for package in $packages; do
|
||||
echo "- $package"
|
||||
# Trim commas, excess spaces, and sort.
|
||||
normalized_packages="$(normalize_package_list "${input_packages}")"
|
||||
|
||||
package_count=$(wc -w <<< "${normalized_packages}")
|
||||
log "Clean installing and caching ${package_count} package(s)."
|
||||
log "Package list:"
|
||||
for package in ${normalized_packages}; do
|
||||
log "- ${package}"
|
||||
done
|
||||
|
||||
echo -n "Updating APT package list..."
|
||||
log "Updating APT package list..."
|
||||
sudo apt-get update > /dev/null
|
||||
echo "done."
|
||||
|
||||
echo "Clean installing and caching $(echo $packages | wc -w) packages..."
|
||||
for package in $packages; do
|
||||
cache_filepath=$cache_dir/$package.tar.gz
|
||||
# Strictly contains the requested packages.
|
||||
manifest_main=""
|
||||
# Contains all packages including dependencies.
|
||||
manifest_all=""
|
||||
|
||||
echo "- $package"
|
||||
echo -n " Installing..."
|
||||
sudo apt-get --yes install $package > /dev/null
|
||||
log "Clean installing and caching ${package_count} packages..."
|
||||
for package in ${normalized_packages}; do
|
||||
read package_name package_ver < <(get_package_name_ver "${package}")
|
||||
|
||||
# Comma delimited name:ver pairs in the main requested packages manifest.
|
||||
manifest_main="${manifest_main}${package_name}:${package_ver},"
|
||||
|
||||
all_packages="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')"
|
||||
dep_packages="$(echo ${all_packages} | grep -v "${package_name}" | tr '\n' ,)"
|
||||
if "${dep_packages}" == ","; then
|
||||
dep_packages="none";
|
||||
fi
|
||||
|
||||
log "- ${package_name}"
|
||||
log " * Version: ${package_ver}"
|
||||
log " * Dependencies: ${dep_packages}"
|
||||
log " * Installing..."
|
||||
# Zero interaction while installing or upgrading the system via apt.
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null
|
||||
echo "done."
|
||||
|
||||
echo -n " Caching to $cache_filepath..."
|
||||
# Pipe all package files (no folders) to Tar.
|
||||
dpkg -L $package |
|
||||
while IFS= read -r f; do
|
||||
if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows
|
||||
done |
|
||||
xargs tar -czf $cache_filepath -C /
|
||||
echo "done."
|
||||
done
|
||||
echo "done."
|
||||
for cache_package in ${all_packages}; do
|
||||
cache_filepath="${cache_dir}/${cache_package}.tar.gz"
|
||||
|
||||
manifest_filepath="$cache_dir/manifest.log"
|
||||
echo -n "Writing package manifest to $manifest_filepath..."
|
||||
manifest=
|
||||
for package in $packages; do
|
||||
manifest=$manifest$package:$(dpkg -s $package | grep Version | awk '{print $2}'),
|
||||
if test ! -f "${cache_filepath}"; then
|
||||
read cache_package_name cache_package_ver < <(get_package_name_ver "${cache_package}")
|
||||
log " * Caching ${cache_package_name} to ${cache_filepath}..."
|
||||
# Pipe all package files (no folders) to Tar.
|
||||
dpkg -L "${cache_package_name}" |
|
||||
while IFS= read -r f; do
|
||||
if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows
|
||||
done |
|
||||
xargs tar -czf "${cache_filepath}" -C /
|
||||
log "done (compressed size $(du -k "${cache_filepath}" | cut -f1))."
|
||||
fi
|
||||
|
||||
# Comma delimited name:ver pairs in the all packages manifest.
|
||||
manifest_all="${manifest_all}${cache_package_name}:${cache_package_ver},"
|
||||
done
|
||||
done
|
||||
# Remove trailing comma.
|
||||
echo ${manifest:0:-1} > $manifest_filepath
|
||||
echo "done."
|
||||
log "done."
|
||||
|
||||
manifest_all_filepath="${cache_dir}/manifest_all.log"
|
||||
log "Writing all packages manifest to ${manifest_all_filepath}..."
|
||||
# Remove trailing comma and write to manifest_all file.
|
||||
echo "${manifest_all:0:-1}" > "${manifest_all_filepath}"
|
||||
log "done."
|
||||
|
||||
manifest_main_filepath="${cache_dir}/manifest_main.log"
|
||||
log "Writing main requested packages manifest to ${manifest_main_filepath}..."
|
||||
# Remove trailing comma and write to manifest_main file.
|
||||
echo "${manifest_main:0:-1}" > "${manifest_main_filepath}"
|
||||
log "done."
|
||||
|
||||
|
|
|
|||
22
lib.sh
Executable file
22
lib.sh
Executable file
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Sort these packages by name and split on commas.
|
||||
function normalize_package_list {
|
||||
local stripped=$(echo "${1}" | sed 's/,//g')
|
||||
# Remove extraneous spaces at the middle, beginning, and end.
|
||||
local trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')"
|
||||
local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')"
|
||||
echo "${sorted}"
|
||||
}
|
||||
|
||||
# Split fully qualified package into name and version
|
||||
function get_package_name_ver {
|
||||
IFS=\= read name ver <<< "${1}"
|
||||
# If version not found in the fully qualified package value.
|
||||
if test -z "${ver}"; then
|
||||
ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')"
|
||||
fi
|
||||
echo "${name}" "${ver}"
|
||||
}
|
||||
|
||||
function log { echo "$(date +%H:%M:%S)" "${@}"; }
|
||||
|
|
@ -3,24 +3,28 @@
|
|||
# Fail on any error.
|
||||
set -e
|
||||
|
||||
# Include library.
|
||||
script_dir="$(dirname -- "$(realpath -- "${0}")")"
|
||||
source "${script_dir}/lib.sh"
|
||||
|
||||
# Directory that holds the cached packages.
|
||||
cache_dir=$1
|
||||
cache_dir="${1}"
|
||||
|
||||
# Root directory to untar the cached packages to.
|
||||
# Typically filesystem root '/' but can be changed for testing.
|
||||
cache_restore_root=$2
|
||||
cache_restore_root="${2}"
|
||||
|
||||
# Indicates that the cache was found.
|
||||
cache_hit=$3
|
||||
cache_hit="${3}"
|
||||
|
||||
# List of the packages to use.
|
||||
packages="${@:4}"
|
||||
|
||||
script_dir=$(dirname $0)
|
||||
script_dir="$(dirname -- "$(realpath -- "${0}")")"
|
||||
|
||||
if [ "$cache_hit" == true ]; then
|
||||
$script_dir/restore_pkgs.sh ~/cache-apt-pkgs $cache_restore_root
|
||||
${script_dir}/restore_pkgs.sh ~/cache-apt-pkgs "${cache_restore_root}"
|
||||
else
|
||||
$script_dir/install_and_cache_pkgs.sh ~/cache-apt-pkgs $packages
|
||||
${script_dir}/install_and_cache_pkgs.sh ~/cache-apt-pkgs ${packages}
|
||||
fi
|
||||
echo ""
|
||||
|
|
|
|||
|
|
@ -1,60 +1,69 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Include library.
|
||||
script_dir="$(dirname -- "$(realpath -- "${0}")")"
|
||||
source "${script_dir}/lib.sh"
|
||||
|
||||
# Directory that holds the cached packages.
|
||||
cache_dir=$1
|
||||
cache_dir="${1}"
|
||||
|
||||
# Version of the cache to create or load.
|
||||
version=$2
|
||||
version="${2}"
|
||||
|
||||
# List of the packages to use.
|
||||
packages=${@:3}
|
||||
input_packages="${@:3}"
|
||||
|
||||
# Trim commas, excess spaces, and sort.
|
||||
packages="$(normalize_package_list "${input_packages}")"
|
||||
|
||||
# Create cache directory so artifacts can be saved.
|
||||
mkdir -p $cache_dir
|
||||
mkdir -p ${cache_dir}
|
||||
|
||||
echo -n "Validating action arguments (version='$version', packages='$packages')...";
|
||||
echo $version | grep -o " " > /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "aborted."
|
||||
echo "Version value '$version' cannot contain spaces." >&2
|
||||
log -n "Validating action arguments (version='${version}', packages='${packages}')...";
|
||||
if grep -q " " <<< "${version}"; then
|
||||
log "aborted."
|
||||
log "Version value '${version}' cannot contain spaces." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$packages" == "" ]; then
|
||||
echo "aborted."
|
||||
echo "Packages argument cannot be empty." >&2
|
||||
|
||||
# Is length of string zero?
|
||||
if test -z "${packages}"; then
|
||||
log "aborted."
|
||||
log "Packages argument cannot be empty." >&2
|
||||
exit 2
|
||||
fi
|
||||
echo "done."
|
||||
log "done."
|
||||
|
||||
echo -n "Verifying packages..."
|
||||
for package in $packages; do
|
||||
escaped=$(echo $package | sed 's/+/\\+/g')
|
||||
apt-cache search ^$escaped$ | grep $package > /dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
versioned_packages=""
|
||||
log -n "Verifying packages..."
|
||||
for package in ${packages}; do
|
||||
if test ! "$(apt show "${package}")"; then
|
||||
echo "aborted."
|
||||
echo "Package '$package' not found." >&2
|
||||
log "Package '${package}' not found." >&2
|
||||
exit 3
|
||||
fi
|
||||
read package_name package_ver < <(get_package_name_ver "${package}")
|
||||
versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}""
|
||||
done
|
||||
echo "done."
|
||||
|
||||
# Abort on any failure at this point.
|
||||
set -e
|
||||
|
||||
echo "Creating cache key..."
|
||||
log "Creating cache key..."
|
||||
|
||||
# Remove package delimiters, sort (requires newline) and then convert back to inline list.
|
||||
normalized_list=$(echo $packages | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ')
|
||||
echo "- Normalized package list is '$normalized_list'."
|
||||
# TODO Can we prove this will happen again?
|
||||
normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")"
|
||||
log "- Normalized package list is '${normalized_versioned_packages}'."
|
||||
|
||||
value=$(echo $normalized_list @ $version)
|
||||
echo "- Value to hash is '$value'."
|
||||
value="${normalized_versioned_packages} @ ${version}"
|
||||
log "- Value to hash is '${value}'."
|
||||
|
||||
key=$(echo $value | md5sum | cut -f1 -d' ')
|
||||
echo "- Value hashed as '$key'."
|
||||
key="$(echo "${value}" | md5sum | /bin/cut -f1 -d' ')"
|
||||
log "- Value hashed as '${key}'."
|
||||
|
||||
echo "done."
|
||||
log "done."
|
||||
|
||||
key_filepath="$cache_dir/cache_key.md5"
|
||||
echo $key > $key_filepath
|
||||
echo "Hash value written to $key_filepath"
|
||||
key_filepath="${cache_dir}/cache_key.md5"
|
||||
echo ${key} > ${key_filepath}
|
||||
log "Hash value written to ${key_filepath}"
|
||||
|
|
|
|||
|
|
@ -3,32 +3,36 @@
|
|||
# Fail on any error.
|
||||
set -e
|
||||
|
||||
# Include library.
|
||||
script_dir="$(dirname -- "$(realpath -- "${0}")")"
|
||||
source "${script_dir}/lib.sh"
|
||||
|
||||
# Directory that holds the cached packages.
|
||||
cache_dir=$1
|
||||
cache_dir="${1}"
|
||||
|
||||
# Root directory to untar the cached packages to.
|
||||
# Typically filesystem root '/' but can be changed for testing.
|
||||
cache_restore_root=$2
|
||||
cache_restore_root="${2}"
|
||||
|
||||
cache_filepaths=$(ls -1 $cache_dir | sort)
|
||||
echo "Found $(echo $cache_filepaths | wc -w) files in the cache."
|
||||
for cache_filepath in $cache_filepaths; do
|
||||
echo "- $(basename $cache_filepath)"
|
||||
cache_filepaths="$(ls -1 "${cache_dir}" | sort)"
|
||||
log "Found $(echo ${cache_filepaths} | wc -w) files in the cache."
|
||||
for cache_filepath in ${cache_filepaths}; do
|
||||
log "- "$(basename ${cache_filepath})""
|
||||
done
|
||||
|
||||
echo "Reading from manifest..."
|
||||
for logline in $(cat $cache_dir/manifest.log | tr ',' '\n' ); do
|
||||
echo "- $(echo $logline | tr ':' ' ')"
|
||||
log "Reading from main requested packages manifest..."
|
||||
for logline in $(cat "${cache_dir}/manifest_main.log" | tr ',' '\n' ); do
|
||||
log "- $(echo "${logline}" | tr ':' ' ')"
|
||||
done
|
||||
echo "done."
|
||||
log "done."
|
||||
|
||||
# Only search for archived results. Manifest and cache key also live here.
|
||||
cache_pkg_filepaths=$(ls -1 $cache_dir/*.tar.gz | sort)
|
||||
cache_pkg_filecount=$(echo $cache_pkg_filepaths | wc -w)
|
||||
echo "Restoring $cache_pkg_filecount packages from cache..."
|
||||
for cache_pkg_filepath in $cache_pkg_filepaths; do
|
||||
echo -n "- $(basename $cache_pkg_filepath) restoring..."
|
||||
sudo tar -xf $cache_pkg_filepath -C $cache_restore_root > /dev/null
|
||||
echo "done."
|
||||
cache_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort)
|
||||
cache_pkg_filecount=$(echo ${cache_pkg_filepaths} | wc -w)
|
||||
log "Restoring ${cache_pkg_filecount} packages from cache..."
|
||||
for cache_pkg_filepath in ${cache_pkg_filepaths}; do
|
||||
log "- $(basename "${cache_pkg_filepath}") restoring..."
|
||||
sudo tar -xf "${cache_pkg_filepath}" -C "${cache_restore_root}" > /dev/null
|
||||
log "done."
|
||||
done
|
||||
echo "done."
|
||||
log "done."
|
||||
|
|
|
|||
Loading…
Reference in a new issue