Copy from staging to dev.

.github/workflows/pub_dev_push_event.yml

@@ -0,0 +1,19 @@
+name: Publish Dev Push Event
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - dev
+
+jobs:
+  publish_event:
+    runs-on: ubuntu-latest
+    name: Publish dev push
+    steps:
+      - run: |
+          curl -i \
+            -X POST \
+            -H "Accept: application/vnd.github.v3+json" \
+            -H "Authorization: token ${{ secrets.PUBLISH_PUSH_TOKEN }}" \
+            https://api.github.com/repos/awalsh128/cache-apt-pkgs-action-ci/dispatches \
+            -d '{"event_type":"dev_push"}'

.github/workflows/pub_master_push_event.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   publish_event:
     runs-on: ubuntu-latest
-    name: Publish staging push
+    name: Publish master push
     steps:
       - run: |
           curl -i \

README.md

@@ -24,8 +24,8 @@ Create a workflow `.yml` file in your repositories `.github/workflows` directory
 ### Outputs
 
 * `cache-hit` - A boolean value to indicate a cache was found for the packages requested.
-* `package-version-list` - The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. \<package1>:<version1\>,\<package2>:\<version2>,...).
+* `package-version-list` - The main requested packages and versions that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. \<package1>:<version1\>,\<package2>:\<version2>,...).
-  
+* `all-package-version-list` - All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. \<package1>:<version1\>,\<package2>:\<version2>,...).
 
 ### Cache scopes
 

action.yml

@@ -26,7 +26,10 @@ outputs:
     # Need to output true and false instead of true and nothing.
     value: ${{ steps.load-cache.outputs.cache-hit || false }}
   package-version-list:
-    description: 'The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. <package>:<version,<package>:<version>).'
+    description: 'The main requested packages and versions that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. <package>:<version,<package>:<version>).'
+    value: ${{ steps.post-cache.outputs.package-version-list }}
+  all-package-version-list:
+    description: 'All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. <package>:<version,<package>:<version>).'
     value: ${{ steps.post-cache.outputs.package-version-list }}
 
 runs:
@@ -42,7 +45,7 @@ runs:
       shell: bash
 
     - id: load-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: ~/cache-apt-pkgs
         key: cache-apt-pkgs_${{ env.CACHE_KEY }}
@@ -54,5 +57,7 @@ runs:
           / \
           "${{ steps.load-cache.outputs.cache-hit }}" \
           ${{ inputs.packages }}
-        echo "::set-output name=package-version-list::$(cat ~/cache-apt-pkgs/manifest.log)"
+        function create_list { local list=$(cat ~/cache-apt-pkgs/manifest_${1}.log | tr '\n' ','); echo ${list:0:-1}; };
+        echo "::set-output name=package-version-list::$(create_list main)"
+        echo "::set-output name=all-package-version-list::$(create_list all)"
       shell: bash

install_and_cache_pkgs.sh

@@ -8,61 +8,72 @@ script_dir="$(dirname -- "$(realpath -- "${0}")")"
 source "${script_dir}/lib.sh"
 
 # Directory that holds the cached packages.
-cache_dir=$1
+cache_dir="${1}"
 
 # List of the packages to use.
 input_packages="${@:2}"
 
 # Trim commas, excess spaces, and sort.
-packages="$(normalize_package_list "${input_packages}")"
+normalized_packages="$(normalize_package_list "${input_packages}")"
 
-package_count=$(echo "${packages}" | wc -w)
+package_count=$(wc -w <<< "${normalized_packages}")
-echo "Clean installing and caching ${package_count} package(s)."
+log "Clean installing and caching ${package_count} package(s)."
-echo "Package list:"
+log "Package list:"
-for package in "${packages}"; do
+for package in ${normalized_packages}; do
-  echo "- ${package}"
+  log "- ${package}"
 done
 
-echo -n "Updating APT package list..."
+log "Updating APT package list..."
 sudo apt-get update > /dev/null
 echo "done."
 
-manifest=""
+# Strictly contains the requested packages.
-echo "Clean installing and caching ${package_count} packages..."
+manifest_main=""
-for package in "${packages}"; do
+# Contains all packages including dependencies.
-  get_package_name_ver "${package}" # -> package_name, package_ver
+manifest_all=""
-  package_deps="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')"
 
-  echo "- ${package_name}"
+log "Clean installing and caching ${package_count} packages..."
-  echo "  * Version: ${package_ver}"
+for package in ${normalized_packages}; do
-  echo "  * Dependencies: ${package_deps}"
+  read package_name package_ver < <(get_package_name_ver "${package}")  
-  echo -n "  * Installing..."
+
+  # Comma delimited name:ver pairs in the main requested packages manifest.
+  manifest_main="${manifest_main}${package_name}:${package_ver},"
+
+  read dep_packages < <(get_dep_packages "${package_name}")
+  if test -z "${dep_packages}"; then
+    dep_packages_text="none";
+  else
+    dep_packages_text="${dep_packages}"
+  fi
+
+  log "- ${package_name}"
+  log "  * Version: ${package_ver}"
+  log "  * Dependencies: ${dep_packages_text}"
+  log "  * Installing..."
   # Zero interaction while installing or upgrading the system via apt.
-  sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null
+  sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package_name}" > /dev/null
   echo "done."
 
-  for cache_package in "${package_deps}"; do
+  for cache_package in ${package_name}:${package_ver} ${dep_packages}; do
     cache_filepath="${cache_dir}/${cache_package}.tar.gz"
 
     if test ! -f "${cache_filepath}"; then
-      get_package_name_ver "${cache_package}" # -> package_name, package_ver      
+      read cache_package_name cache_package_ver < <(get_package_name_ver "${cache_package}")
-      echo -n "  Caching ${package_name} to ${cache_filepath}..."
+      log "  * Caching ${cache_package_name} to ${cache_filepath}..."
       # Pipe all package files (no folders) to Tar.
-      dpkg -L "${package_name}" |
+      dpkg -L "${cache_package_name}" |
         while IFS= read -r f; do     
-          if test -f $f; then echo "${f:1}"; fi;  #${f:1} removes the leading slash that Tar disallows
+          if test -f $f || test -L $f; then echo "${f:1}"; fi;  #${f:1} removes the leading slash that Tar disallows
         done | 
         xargs tar -czf "${cache_filepath}" -C /      
-      echo "done."
+      log "done (compressed size $(du -h "${cache_filepath}" | cut -f1))."
-      # Add package to manifest
-      manifest="${manifest}${package_name}:$(dpkg -s "${package_name}" | grep Version | awk '{print $2}'),"
     fi
-  done
-done
-echo "done."
 
-manifest_filepath="${cache_dir}/manifest.log"
+    # Comma delimited name:ver pairs in the all packages manifest.
-echo -n "Writing package manifest to ${manifest_filepath}..."
+    manifest_all="${manifest_all}${cache_package_name}:${cache_package_ver},"
-# Remove trailing comma and write to manifest file.
+  done  
-echo "${manifest:0:-1}" > "${manifest_filepath}"
+done
-echo "done."
+log "done."
+
+write_manifest "all" "${manifest_all}" "${cache_dir}/manifest_all.log"
+write_manifest "main" "${manifest_main}" "${cache_dir}/manifest_main.log"

lib.sh

@@ -1,19 +1,37 @@
-#!/bin/bash -x
+#!/bin/bash
 
 # Sort these packages by name and split on commas.
 function normalize_package_list {
-  stripped="$(echo ${1} | sed 's/,//g')"
+  local stripped=$(echo "${1}" | sed 's/,//g')
   # Remove extraneous spaces at the middle, beginning, and end.
-  trimmed="$(echo ${stripped} | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')"  
+  local trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')"
-  echo "${trimmed}" | sort
+  local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')"
+  echo "${sorted}"  
+}
+
+# Gets a package list of dependencies as common delimited pairs
+#   <name>:<version>,<name:version>...
+function get_dep_packages {  
+  echo $(apt-get install --dry-run --yes "${1}" | \
+    grep "^Inst" | sort | awk '{print $2 $3}' | \
+    tr '(' ':' | grep -v "${1}:")
 }
 
 # Split fully qualified package into name and version
 function get_package_name_ver {
-  IFS=\= read name ver <<< "${1}"
+  IFS=\: read name ver <<< "${1}"
   # If version not found in the fully qualified package value.
   if test -z "${ver}"; then
     ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')"
   fi
-  echo 'package_name="${name}"; package_ver="${ver}"'
+  echo "${name}" "${ver}"
+}
+
+function log { echo "$(date +%H:%M:%S)" "${@}"; }
+
+function write_manifest {  
+  log "Writing ${1} packages manifest to ${3}..."  
+  # 0:-1 to remove trailing comma, delimit by newline and sort
+  echo "${2:0:-1}" | tr ',' '\n' | sort > ${3}
+  log "done."
 }

post_cache_action.sh

@@ -3,24 +3,28 @@
 # Fail on any error.
 set -e
 
+# Include library.
+script_dir="$(dirname -- "$(realpath -- "${0}")")"
+source "${script_dir}/lib.sh"
+
 # Directory that holds the cached packages.
-cache_dir=$1
+cache_dir="${1}"
 
 # Root directory to untar the cached packages to.
 # Typically filesystem root '/' but can be changed for testing.
-cache_restore_root=$2
+cache_restore_root="${2}"
 
 # Indicates that the cache was found.
-cache_hit=$3
+cache_hit="${3}"
 
 # List of the packages to use.
 packages="${@:4}"
 
-script_dir=$(dirname $0)
+script_dir="$(dirname -- "$(realpath -- "${0}")")"
 
 if [ "$cache_hit" == true ]; then
-  $script_dir/restore_pkgs.sh ~/cache-apt-pkgs $cache_restore_root
+  ${script_dir}/restore_pkgs.sh ~/cache-apt-pkgs "${cache_restore_root}"
 else
-  $script_dir/install_and_cache_pkgs.sh ~/cache-apt-pkgs $packages
+  ${script_dir}/install_and_cache_pkgs.sh ~/cache-apt-pkgs ${packages}
 fi
 echo ""

pre_cache_action.sh

@@ -8,7 +8,7 @@ source "${script_dir}/lib.sh"
 cache_dir="${1}"
 
 # Version of the cache to create or load.
-cache_version="${2}"
+version="${2}"
 
 # List of the packages to use.
 input_packages="${@:3}"
@@ -17,57 +17,53 @@ input_packages="${@:3}"
 packages="$(normalize_package_list "${input_packages}")"
 
 # Create cache directory so artifacts can be saved.
-mkdir -p "${cache_dir}"
+mkdir -p ${cache_dir}
 
-echo -n "Validating action arguments (version='${cache_version}', packages='${packages}')...";
+log -n "Validating action arguments (version='${version}', packages='${packages}')...";
-if grep -q " " <<< "${cache_version}"; then
+if grep -q " " <<< "${version}"; then
-  echo "aborted." 
+  log "aborted." 
-  echo "Version value '${cache_version}' cannot contain spaces." >&2
+  log "Version value '${version}' cannot contain spaces." >&2
   exit 1
 fi
 
 # Is length of string zero?
 if test -z "${packages}"; then
-  echo "aborted." 
+  log "aborted."
-  echo "Packages argument cannot be empty." >&2
+  log "Packages argument cannot be empty." >&2
   exit 2
 fi
-echo "done."
+log "done."
-
-echo -n "Updating APT package list..."
-sudo apt-get update > /dev/null
-echo "done."
 
 versioned_packages=""
-echo -n "Verifying packages..."
+log -n "Verifying packages..."
 for package in ${packages}; do 
   if test ! "$(apt show "${package}")"; then
     echo "aborted."
-    echo "Package '${package}' not found." >&2
+    log "Package '${package}' not found." >&2
     exit 3
   fi
-  get_package_name_ver "${package}" # -> package_name, package_ver  
+  read package_name package_ver < <(get_package_name_ver "${package}")
-  versioned_packages="${versioned_packages} ${package_name}=${package_ver}"
+  versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}""
 done
 echo "done."
 
 # Abort on any failure at this point.
 set -e
 
-echo "Creating cache key..."
+log "Creating cache key..."
 
 # TODO Can we prove this will happen again?
 normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")"
-echo "- Normalized package list is '${normalized_versioned_packages}'."
+log "- Normalized package list is '${normalized_versioned_packages}'."
 
-value="$(echo "${normalized_versioned_packages} @ ${cache_version}")"
+value="${normalized_versioned_packages} @ ${version}"
-echo "- Value to hash is '${value}'."
+log "- Value to hash is '${value}'."
 
 key="$(echo "${value}" | md5sum | /bin/cut -f1 -d' ')"
-echo "- Value hashed as '${key}'."
+log "- Value hashed as '${key}'."
 
-echo "done."
+log "done."
 
 key_filepath="${cache_dir}/cache_key.md5"
-echo "${key}" > "${key_filepath}"
+echo ${key} > ${key_filepath}
-echo "Hash value written to ${key_filepath}"
+log "Hash value written to ${key_filepath}"

restore_pkgs.sh

@@ -3,32 +3,36 @@
 # Fail on any error.
 set -e
 
+# Include library.
+script_dir="$(dirname -- "$(realpath -- "${0}")")"
+source "${script_dir}/lib.sh"
+
 # Directory that holds the cached packages.
-cache_dir=$1
+cache_dir="${1}"
 
 # Root directory to untar the cached packages to.
 # Typically filesystem root '/' but can be changed for testing.
-cache_restore_root=$2
+cache_restore_root="${2}"
 
-cache_filepaths=$(ls -1 $cache_dir | sort)
+cache_filepaths="$(ls -1 "${cache_dir}" | sort)"
-echo "Found $(echo $cache_filepaths | wc -w) files in the cache."
+log "Found $(echo ${cache_filepaths} | wc -w) files in the cache."
-for cache_filepath in $cache_filepaths; do
+for cache_filepath in ${cache_filepaths}; do
-  echo "- $(basename $cache_filepath)"
+  log "- "$(basename ${cache_filepath})""
 done
 
-echo "Reading from manifest..."
+log "Reading from main requested packages manifest..."
-for logline in "$(cat "${cache_dir}/manifest.log" | tr ',' '\n' )"; do
+for logline in $(cat "${cache_dir}/manifest_main.log" | tr ',' '\n' ); do
-  echo "- $(echo "${logline}" | tr ':' ' ')"
+  log "- $(echo "${logline}" | tr ':' ' ')"
 done
-echo "done."
+log "done."
 
 # Only search for archived results. Manifest and cache key also live here.
-cache_pkg_filepaths=$(ls -1 $cache_dir/*.tar.gz | sort)
+cache_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort)
-cache_pkg_filecount=$(echo $cache_pkg_filepaths | wc -w)
+cache_pkg_filecount=$(echo ${cache_pkg_filepaths} | wc -w)
-echo "Restoring $cache_pkg_filecount packages from cache..."
+log "Restoring ${cache_pkg_filecount} packages from cache..."
-for cache_pkg_filepath in $cache_pkg_filepaths; do
+for cache_pkg_filepath in ${cache_pkg_filepaths}; do
-  echo -n "- $(basename $cache_pkg_filepath) restoring..."
+  log "- $(basename "${cache_pkg_filepath}") restoring..."
-  sudo tar -xf $cache_pkg_filepath -C $cache_restore_root > /dev/null
+  sudo tar -xf "${cache_pkg_filepath}" -C "${cache_restore_root}" > /dev/null
-  echo "done."
+  log "done."
 done
-echo "done."
+log "done."