From 13d7657f54a7adb371577bea52e00d8e92f9a78a Mon Sep 17 00:00:00 2001 From: jroddev Date: Sun, 13 Feb 2022 16:54:39 +1100 Subject: [PATCH 1/3] g++ install is failing because + is treated as a regex control character --- pre_cache_action.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 51a5b78..c4f764b 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -28,7 +28,8 @@ echo "done." echo -n "Verifying packages..." for package in $packages; do - apt-cache search ^$package$ | grep $package > /dev/null + escaped=$(echo $package | sed 's/+/\\+/g') + apt-cache search ^$escaped$ | grep $package > /dev/null if [ $? -ne 0 ]; then echo "aborted." echo "Package '$package' not found." >&2 From a56b21237f73b90875be42d33eb7afc4ccd2c46a Mon Sep 17 00:00:00 2001 From: Nate Bohman Date: Wed, 16 Mar 2022 11:15:25 -0600 Subject: [PATCH 2/3] Check APT Package Versions Make sure to keep the cache up to date as package versions change upstream. Also, allow specific package versions to be selected. --- install_and_cache_pkgs.sh | 3 +- pre_cache_action.sh | 62 ++++++++++++++++++++++++--------------- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index bbf54dc..20f3974 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -31,7 +31,7 @@ for package in $packages; do echo -n " Caching to $cache_filepath..." # Pipe all package files (no folders) to Tar. - dpkg -L $package | + dpkg -L "$(echo "${package}" | cut -d"=" -f1)" | while IFS= read -r f; do if test -f $f; then echo ${f:1}; fi; #${f:1} removes the leading slash that Tar disallows done | @@ -44,6 +44,7 @@ manifest_filepath="$cache_dir/manifest.log" echo -n "Writing package manifest to $manifest_filepath..." manifest= for package in $packages; do + package="$(echo "${package}" | cut -d"=" -f1)" manifest=$manifest$package:$(dpkg -s $package | grep Version | awk '{print $2}'), done # Remove trailing comma. diff --git a/pre_cache_action.sh b/pre_cache_action.sh index c4f764b..a464edd 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -1,40 +1,56 @@ #!/bin/bash # Directory that holds the cached packages. -cache_dir=$1 +cache_dir="${1}" # Version of the cache to create or load. -version=$2 +version="${2}" # List of the packages to use. -packages=${@:3} +packages="${@:3}" + +# Sort these packages by name and split on commas. +packages=$(echo "${packages}" | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') # Create cache directory so artifacts can be saved. -mkdir -p $cache_dir +mkdir -p "${cache_dir}" -echo -n "Validating action arguments (version='$version', packages='$packages')..."; -echo $version | grep -o " " > /dev/null -if [ $? -eq 0 ]; then +echo -n "Validating action arguments (version='${version}', packages='${packages}')..."; + +if echo "${version}" | grep -q " " > /dev/null; then echo "aborted." - echo "Version value '$version' cannot contain spaces." >&2 + echo "Version value '${version}' cannot contain spaces." >&2 exit 1 fi -if [ "$packages" == "" ]; then +if [ "${packages}" == "" ]; then echo "aborted." echo "Packages argument cannot be empty." >&2 exit 2 fi echo "done." +echo -n "Updating APT package list..." +sudo apt-get update > /dev/null +echo "done." + echo -n "Verifying packages..." -for package in $packages; do - escaped=$(echo $package | sed 's/+/\\+/g') - apt-cache search ^$escaped$ | grep $package > /dev/null - if [ $? -ne 0 ]; then +for package in ${packages}; do + if echo "${package}" | grep -q "="; then + pkg_name=$(echo "${package}" | cut -d "=" -f1) + pkg_ver=$(echo "${package}" | cut -d "=" -f2) + else + pkg_name="${package}" + fi + apt_show=$(apt show "${package}") + if echo ${apt_show} | grep -qi "No packages found" > /dev/null; then echo "aborted." - echo "Package '$package' not found." >&2 + echo "Package '${package}' not found." >&2 exit 3 fi + if [ -z "${pkg_ver}" ]; then + pkg_ver=$(echo "${apt_show}" | grep -Po "(?<=Version: )[^\s]+") + fi + package_list="${package_list} ${pkg_name}=${pkg_ver}" done echo "done." @@ -43,18 +59,18 @@ set -e echo "Creating cache key..." -# Remove package delimiters, sort (requires newline) and then convert back to inline list. -normalized_list=$(echo $packages | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') -echo "- Normalized package list is '$normalized_list'." +# Remove extraneous spaces +package_list="$(echo "${package_list}" | sed 's/\s\+/ /g;s/^\s\+//g;s/\s\+$//g')" +echo "- Normalized package list is '$package_list'." -value=$(echo $normalized_list @ $version) -echo "- Value to hash is '$value'." +value=$(echo "${package_list} @ ${version}") +echo "- Value to hash is '${value}'." -key=$(echo $value | md5sum | /bin/cut -f1 -d' ') +key=$(echo "${value}" | md5sum | /bin/cut -f1 -d' ') echo "- Value hashed as '$key'." echo "done." -key_filepath="$cache_dir/cache_key.md5" -echo $key > $key_filepath -echo "Hash value written to $key_filepath" +key_filepath="${cache_dir}/cache_key.md5" +echo "${key}" > "${key_filepath}" +echo "Hash value written to ${key_filepath}" From 6d622023ef77ba5f64a9a939fc4df08c3bb048ae Mon Sep 17 00:00:00 2001 From: Nate Bohman Date: Wed, 16 Mar 2022 14:36:54 -0600 Subject: [PATCH 3/3] Also Cache Non-Explicitly Listed APT PKGs Apt installs more packages than just those explicitly listed. This can lead to issues of missing files when the cache is restored. --- action.yml | 4 +++ install_and_cache_pkgs.sh | 67 +++++++++++++++++++++++---------------- post_cache_action.sh | 14 ++++---- pre_cache_action.sh | 22 ++++++------- restore_pkgs.sh | 28 ++++++++-------- 5 files changed, 76 insertions(+), 59 deletions(-) diff --git a/action.yml b/action.yml index e717de5..38d4e13 100644 --- a/action.yml +++ b/action.yml @@ -25,6 +25,9 @@ outputs: # This compound expression is needed because lhs can be empty. # Need to output true and false instead of true and nothing. value: ${{ steps.load-cache.outputs.cache-hit || false }} + cache-key: + description: 'The key where the cache is stored with packages, their versions, and the cache version (i.e. package=version package=version @ 1).' + value: ${{ steps.pre-cache.outputs.cache-key }} package-version-list: description: 'The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. ::).' value: ${{ steps.post-cache.outputs.package-version-list }} @@ -39,6 +42,7 @@ runs: "${{ inputs.version }}" \ ${{ inputs.packages }} echo "CACHE_KEY=$(cat ~/cache-apt-pkgs/cache_key.md5)" >> $GITHUB_ENV + echo "::set-output name=cache-key::$(cat ~/cache-apt-pkgs/cache_key.md5)" shell: bash - id: load-cache diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 20f3974..ca6c838 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -4,49 +4,62 @@ set -e # Directory that holds the cached packages. -cache_dir=$1 +cache_dir="${1}" # List of the packages to use. packages="${@:2}" -package_count=$(echo $packages | wc -w) -echo "Clean installing and caching $package_count package(s)." +# Sort these packages by name and split on commas. +packages=$(echo "${packages}" | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') + +# Remove extraneous spaces +packages="$(echo "${packages}" | sed 's/\s\+/ /g;s/^\s\+//g;s/\s\+$//g')" + +package_count=$(echo "${packages}" | wc -w) +echo "Clean installing and caching ${package_count} package(s)." echo "Package list:" -for package in $packages; do - echo "- $package" +for package in ${packages}; do + echo "- ${package}" done echo -n "Updating APT package list..." sudo apt-get update > /dev/null echo "done." -echo "Clean installing and caching $(echo $packages | wc -w) packages..." -for package in $packages; do - cache_filepath=$cache_dir/$package.tar.gz - - echo "- $package" +manifest= +echo "Clean installing and caching ${package_count} packages..." +for package in ${packages}; do + echo "- ${package}" echo -n " Installing..." - sudo apt-get --yes install $package > /dev/null + + # Gather a list of all packages apt installs to make this package work + required_packages=$(apt-get install --dry-run --yes "${package}" | grep -Po "(?<=Inst )[^\s]+" || echo "${package}") + echo "Package ${package} installs the following packages: ${required_packages//$'\n'/, }" + sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null + echo "done." - echo -n " Caching to $cache_filepath..." - # Pipe all package files (no folders) to Tar. - dpkg -L "$(echo "${package}" | cut -d"=" -f1)" | - while IFS= read -r f; do - if test -f $f; then echo ${f:1}; fi; #${f:1} removes the leading slash that Tar disallows - done | - xargs tar -czf $cache_filepath -C / - echo "done." + for cache_package in ${required_packages}; do + cache_filepath="${cache_dir}/${cache_package}.tar.gz" + if [ ! -f "${cache_filepath}" ]; then + package_name="$(echo "${cache_package}" | cut -d"=" -f1)" + echo -n " Caching ${package_name} to ${cache_filepath}..." + # Pipe all package files (no folders) to Tar. + dpkg -L "${package_name}" | + while IFS= read -r f; do + if test -f $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows + done | + xargs tar -czf "${cache_filepath}" -C / + echo "done." + # Add package to manifest + manifest="${manifest}${package_name}:$(dpkg -s "${package_name}" | grep Version | awk '{print $2}')," + fi + done done echo "done." -manifest_filepath="$cache_dir/manifest.log" -echo -n "Writing package manifest to $manifest_filepath..." -manifest= -for package in $packages; do - package="$(echo "${package}" | cut -d"=" -f1)" - manifest=$manifest$package:$(dpkg -s $package | grep Version | awk '{print $2}'), -done +manifest_filepath="${cache_dir}/manifest.log" +echo -n "Writing package manifest to ${manifest_filepath}..." # Remove trailing comma. -echo ${manifest:0:-1} > $manifest_filepath +echo ${manifest:0:-1} > ${manifest_filepath} echo "done." \ No newline at end of file diff --git a/post_cache_action.sh b/post_cache_action.sh index 5503013..0bb091f 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -4,23 +4,23 @@ set -e # Directory that holds the cached packages. -cache_dir=$1 +cache_dir="${1}" # Root directory to untar the cached packages to. # Typically filesystem root '/' but can be changed for testing. -cache_restore_root=$2 +cache_restore_root="${2}" # Indicates that the cache was found. -cache_hit=$3 +cache_hit="${3}" # List of the packages to use. packages="${@:4}" -script_dir=$(dirname $0) +script_dir="$(dirname -- "$(realpath -- "${0}")")" -if [ "$cache_hit" == true ]; then - $script_dir/restore_pkgs.sh ~/cache-apt-pkgs $cache_restore_root +if [ "${cache_hit}" == true ]; then + "${script_dir}/restore_pkgs.sh" ~/cache-apt-pkgs "${cache_restore_root}" else - $script_dir/install_and_cache_pkgs.sh ~/cache-apt-pkgs $packages + "${script_dir}/install_and_cache_pkgs.sh" ~/cache-apt-pkgs "${packages}" fi echo "" diff --git a/pre_cache_action.sh b/pre_cache_action.sh index a464edd..d9dd664 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -4,7 +4,7 @@ cache_dir="${1}" # Version of the cache to create or load. -version="${2}" +cache_version="${2}" # List of the packages to use. packages="${@:3}" @@ -15,11 +15,11 @@ packages=$(echo "${packages}" | sed 's/[\s,]+/ /g' | tr ' ' '\n' | sort | tr '\n # Create cache directory so artifacts can be saved. mkdir -p "${cache_dir}" -echo -n "Validating action arguments (version='${version}', packages='${packages}')..."; +echo -n "Validating action arguments (version='${cache_version}', packages='${packages}')..."; -if echo "${version}" | grep -q " " > /dev/null; then +if echo "${cache_version}" | grep -q " " > /dev/null; then echo "aborted." - echo "Version value '${version}' cannot contain spaces." >&2 + echo "Version value '${cache_version}' cannot contain spaces." >&2 exit 1 fi if [ "${packages}" == "" ]; then @@ -36,10 +36,10 @@ echo "done." echo -n "Verifying packages..." for package in ${packages}; do if echo "${package}" | grep -q "="; then - pkg_name=$(echo "${package}" | cut -d "=" -f1) - pkg_ver=$(echo "${package}" | cut -d "=" -f2) + package_name=$(echo "${package}" | cut -d "=" -f1) + package_ver=$(echo "${package}" | cut -d "=" -f2) else - pkg_name="${package}" + package_name="${package}" fi apt_show=$(apt show "${package}") if echo ${apt_show} | grep -qi "No packages found" > /dev/null; then @@ -47,10 +47,10 @@ for package in ${packages}; do echo "Package '${package}' not found." >&2 exit 3 fi - if [ -z "${pkg_ver}" ]; then - pkg_ver=$(echo "${apt_show}" | grep -Po "(?<=Version: )[^\s]+") + if [ -z "${package_ver}" ]; then + package_ver=$(echo "${apt_show}" | grep -Po "(?<=Version: )[^\s]+") fi - package_list="${package_list} ${pkg_name}=${pkg_ver}" + package_list="${package_list} ${package_name}=${package_ver}" done echo "done." @@ -63,7 +63,7 @@ echo "Creating cache key..." package_list="$(echo "${package_list}" | sed 's/\s\+/ /g;s/^\s\+//g;s/\s\+$//g')" echo "- Normalized package list is '$package_list'." -value=$(echo "${package_list} @ ${version}") +value=$(echo "${package_list} @ ${cache_version}") echo "- Value to hash is '${value}'." key=$(echo "${value}" | md5sum | /bin/cut -f1 -d' ') diff --git a/restore_pkgs.sh b/restore_pkgs.sh index ab87879..4fdbcc6 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -4,31 +4,31 @@ set -e # Directory that holds the cached packages. -cache_dir=$1 +cache_dir="${1}" # Root directory to untar the cached packages to. # Typically filesystem root '/' but can be changed for testing. -cache_restore_root=$2 +cache_restore_root="${2}" -cache_filepaths=$(ls -1 $cache_dir | sort) -echo "Found $(echo $cache_filepaths | wc -w) files in the cache." -for cache_filepath in $cache_filepaths; do - echo "- $(basename $cache_filepath)" +cache_filepaths="$(ls -1 "${cache_dir}" | sort)" +echo "Found $(echo "${cache_filepaths}" | wc -w) files in the cache." +for cache_filepath in ${cache_filepaths}; do + echo "- $(basename "${cache_filepath}")" done echo "Reading from manifest..." -for logline in $(cat $cache_dir/manifest.log | tr ',' '\n' ); do - echo "- $(echo $logline | tr ':' ' ')" +for logline in $(cat "${cache_dir}/manifest.log" | tr ',' '\n' ); do + echo "- $(echo "${logline}" | tr ':' ' ')" done echo "done." # Only search for archived results. Manifest and cache key also live here. -cache_pkg_filepaths=$(ls -1 $cache_dir/*.tar.gz | sort) -cache_pkg_filecount=$(echo $cache_pkg_filepaths | wc -w) -echo "Restoring $cache_pkg_filecount packages from cache..." -for cache_pkg_filepath in $cache_pkg_filepaths; do - echo -n "- $(basename $cache_pkg_filepath) restoring..." - sudo tar -xf $cache_pkg_filepath -C $cache_restore_root > /dev/null +cache_pkg_filepaths=$(ls -1 ${cache_dir}/*.tar.gz | sort) +cache_pkg_filecount=$(echo "${cache_pkg_filepaths}" | wc -w) +echo "Restoring ${cache_pkg_filecount} packages from cache..." +for cache_pkg_filepath in ${cache_pkg_filepaths}; do + echo -n "- $(basename "${cache_pkg_filepath}") restoring..." + sudo tar -xf ${cache_pkg_filepath} -C ${cache_restore_root} > /dev/null echo "done." done echo "done."