Merge branch 'dev' into master

install_and_cache_pkgs.sh

@@ -6,7 +6,7 @@ set -e
 # Debug mode for diagnosing issues.
 # Setup first before other operations.
 debug="${2}"
-test ${debug} == "true" && set -x
+test "${debug}" = "true" && set -x
 
 # Include library.
 script_dir="$(dirname -- "$(realpath -- "${0}")")"
@@ -18,20 +18,23 @@ cache_dir="${1}"
 # List of the packages to use.
 input_packages="${@:3}"
 
-# Trim commas, excess spaces, and sort.
+# Trim commas, excess spaces, sort, and version syntax.
-normalized_packages="$(normalize_package_list "${input_packages}")"
+#
+# NOTE: Unless specified, all APT package listings of name and version use
+# colon delimited and not equals delimited syntax (i.e. <name>[:=]<ver>).
+packages="$(get_normalized_package_list "${input_packages}")"
 
-package_count=$(wc -w <<< "${normalized_packages}")
+package_count=$(wc -w <<< "${packages}")
 log "Clean installing and caching ${package_count} package(s)."
 
 log_empty_line
 
 manifest_main=""
 log "Package list:"
-for package in ${normalized_packages}; do
+for package in ${packages}; do
   read package_name package_ver < <(get_package_name_ver "${package}")
-  manifest_main="${manifest_main}${package_name}:${package_ver},"
+  manifest_main="${manifest_main}${package_name}=${package_ver},"  
-  log "- ${package_name}:${package_ver}"
+  log "- ${package_name} (${package_ver})"
 done
 write_manifest "main" "${manifest_main}" "${cache_dir}/manifest_main.log"
 
@@ -65,7 +68,7 @@ install_log_filepath="${cache_dir}/install.log"
 
 log "Clean installing ${package_count} packages..."
 # Zero interaction while installing or upgrading the system via apt.
-sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${normalized_packages} > "${install_log_filepath}"
+sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${packages} > "${install_log_filepath}"
 log "done"
 log "Installation log written to ${install_log_filepath}"
 
@@ -74,34 +77,35 @@ log_empty_line
 installed_packages=$(get_installed_packages "${install_log_filepath}")
 log "Installed package list:"
 for installed_package in ${installed_packages}; do
-  log "- ${installed_package}"
+  # Reformat for human friendly reading.  
+  log "- $(echo ${installed_package} | awk -F\= '{print $1" ("$2")"}')"
 done
 
 log_empty_line
 
-installed_package_count=$(wc -w <<< "${installed_packages}")
+installed_packages_count=$(wc -w <<< "${installed_packages}")
-log "Caching ${installed_package_count} installed packages..."
+log "Caching ${installed_packages_count} installed packages..."
 for installed_package in ${installed_packages}; do
   cache_filepath="${cache_dir}/${installed_package}.tar"
 
   # Sanity test in case APT enumerates duplicates.
   if test ! -f "${cache_filepath}"; then
-    read installed_package_name installed_package_ver < <(get_package_name_ver "${installed_package}")
+    read package_name package_ver < <(get_package_name_ver "${installed_package}")
-    log "  * Caching ${installed_package_name} to ${cache_filepath}..."
+    log "  * Caching ${package_name} to ${cache_filepath}..."
 
     # Pipe all package files (no folders) and installation control data to Tar.
-    { dpkg -L "${installed_package_name}" \
+    { dpkg -L "${package_name}" \
-      & get_install_filepath "" "${package_name}" "preinst" \
+      & get_install_script_filepath "" "${package_name}" "preinst" \
-      & get_install_filepath "" "${package_name}" "postinst"; } |
+      & get_install_script_filepath "" "${package_name}" "postinst"; } |
       while IFS= read -r f; do test -f "${f}" -o -L "${f}" && get_tar_relpath "${f}"; done |
-      xargs -I {} echo \"{}\" |
+      xargs -I {} echo \'{}\' | # Single quotes ensure literals like backslash get captured.
       sudo xargs tar -cf "${cache_filepath}" -C /
 
     log "    done (compressed size $(du -h "${cache_filepath}" | cut -f1))."
   fi
 
   # Comma delimited name:ver pairs in the all packages manifest.
-  manifest_all="${manifest_all}${installed_package_name}:${installed_package_ver},"
+  manifest_all="${manifest_all}${package_name}=${package_ver},"
 done
 log "done (total cache size $(du -h ${cache_dir} | tail -1 | awk '{print $1}'))"
 

lib.sh

@@ -11,9 +11,9 @@
 #   Filepath of the install script, otherwise an empty string.
 ###############################################################################
 function execute_install_script {
-  local package_name=$(basename ${2} | awk -F\: '{print $1}')  
+  local package_name=$(basename ${2} | awk -F\= '{print $1}')  
   local install_script_filepath=$(\
-    get_install_filepath "${1}" "${package_name}" "${3}")
+    get_install_script_filepath "${1}" "${package_name}" "${3}")
   if test ! -z "${install_script_filepath}"; then
     log "- Executing ${install_script_filepath}..."
     # Don't abort on errors; dpkg-trigger will error normally since it is
@@ -23,21 +23,39 @@ function execute_install_script {
   fi
 }
 
+###############################################################################
+# Gets the Debian install script filepath.
+# Arguments:
+#   Root directory to search from.
+#   Name of the unqualified package to search for.
+#   Extension of the installation script (preinst, postinst)
+# Returns:
+#   Filepath of the script file, otherwise an empty string.
+###############################################################################
+function get_install_script_filepath {
+  # Filename includes arch (e.g. amd64).
+  local filepath="$(\
+    ls -1 ${1}var/lib/dpkg/info/${2}*.${3} 2> /dev/null \
+    | grep -E ${2}'(:.*)?.'${3} | head -1 || true)"
+  test "${filepath}" && echo "${filepath}"
+}
+
 ###############################################################################
 # Gets a list of installed packages from a Debian package installation log.
 # Arguments:
 #   The filepath of the Debian install log.
 # Returns:
-#   The list of space delimited pairs with each pair colon delimited.
+#   The list of colon delimited action syntax pairs with each pair equals
-#   <name>:<version> <name:version>...
+#   delimited. <name>:<version> <name>:<version>...
 ###############################################################################
 function get_installed_packages {   
   local install_log_filepath="${1}"
   local regex="^Unpacking ([^ :]+)([^ ]+)? (\[[^ ]+\]\s)?\(([^ )]+)"  
   local dep_packages=""  
   while read -r line; do
+    # ${regex} should be unquoted since it isn't a literal.
     if [[ "${line}" =~ ${regex} ]]; then
-      dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[4]} "      
+      dep_packages="${dep_packages}${BASH_REMATCH[1]}=${BASH_REMATCH[4]} "      
     else
       log_err "Unable to parse package name and version from \"${line}\""
       exit 2
@@ -51,14 +69,16 @@ function get_installed_packages {
 }
 
 ###############################################################################
-# Splits a fully qualified package into name and version.
+# Splits a fully action syntax APT package into the name and version.
 # Arguments:
-#   The colon delimited package pair or just the package name.
+#   The action syntax colon delimited package pair or just the package name.
 # Returns:
 #   The package name and version pair.
 ###############################################################################
 function get_package_name_ver {
-  IFS=\: read name ver <<< "${1}"
+  local ORIG_IFS="${IFS}"
+  IFS=\= read name ver <<< "${1}"
+  IFS="${ORIG_IFS}"
   # If version not found in the fully qualified package value.
   if test -z "${ver}"; then
     ver="$(grep "Version:" <<< "$(apt-cache show ${name})" | awk '{print $2}')"
@@ -67,32 +87,20 @@ function get_package_name_ver {
 }
 
 ###############################################################################
-# Gets the package name from the cached package filepath in the
+# Sorts given packages by name and split on commas and/or spaces.
-# path/to/cache/dir/<name>:<version>.tar format.
 # Arguments:
-#   Filepath to the cached packaged.
+#   The comma and/or space delimited list of packages.
 # Returns:
-#   The package name.
+#   Sorted list of space delimited packages.
 ###############################################################################
-function get_package_name_from_cached_filepath {
+function get_normalized_package_list {
-  basename ${cached_pkg_filepath} | awk -F\: '{print $1}'
+  # Remove commas, and block scalar folded backslashes.
-}
+  local stripped=$(echo "${1}" | sed 's/[,\]/ /g')
-
+  # Remove extraneous spaces at the middle, beginning, and end.
-###############################################################################
+  local trimmed="$(\
-# Gets the Debian install script file location.
+    echo "${stripped}" \
-# Arguments:
+    | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')"
-#   Root directory to search from.
+  echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' '
-#   Name of the unqualified package to search for.
-#   Extension of the installation script (preinst, postinst)
-# Returns:
-#   Filepath of the script file, otherwise an empty string.
-###############################################################################
-function get_install_filepath {
-  # Filename includes arch (e.g. amd64).
-  local filepath="$(\
-    ls -1 ${1}var/lib/dpkg/info/${2}*.${3} 2> /dev/null \
-    | grep -E ${2}'(:.*)?.'${3} | head -1 || true)"
-  test "${filepath}" && echo "${filepath}"
 }
 
 ###############################################################################
@@ -117,24 +125,6 @@ function log_err { >&2 echo "$(date +%H:%M:%S)" "${@}"; }
 
 function log_empty_line { echo ""; }
 
-###############################################################################
-# Sorts given packages by name and split on commas and/or spaces.
-# Arguments:
-#   The comma and/or space delimited list of packages.
-# Returns:
-#   Sorted list of space delimited packages.
-###############################################################################
-function normalize_package_list {
-  # Remove commas, and block scalar folded backslashes.
-  local stripped=$(echo "${1}" | sed 's/[,\]/ /g')
-  # Remove extraneous spaces at the middle, beginning, and end.
-  local trimmed="$(\
-    echo "${stripped}" \
-    | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')"
-  local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')"
-  echo "${sorted}"  
-}
-
 ###############################################################################
 # Validates an argument to be of a boolean value.
 # Arguments:

post_cache_action.sh

@@ -23,12 +23,12 @@ execute_install_scripts="${4}"
 
 # Debug mode for diagnosing issues.
 debug="${5}"
-test ${debug} == "true" && set -x
+test "${debug}" = "true" && set -x
 
 # List of the packages to use.
 packages="${@:6}"
 
-if [ "$cache_hit" == true ]; then
+if test "${cache_hit}" = "true"; then
   ${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}"
 else
   ${script_dir}/install_and_cache_pkgs.sh "${cache_dir}" "${debug}" ${packages}

pre_cache_action.sh

@@ -28,7 +28,7 @@ debug="${4}"
 input_packages="${@:5}"
 
 # Trim commas, excess spaces, and sort.
-packages="$(normalize_package_list "${input_packages}")"
+packages="$(get_normalized_package_list "${input_packages}")"
 
 # Create cache directory so artifacts can be saved.
 mkdir -p ${cache_dir}
@@ -56,7 +56,7 @@ log_empty_line
 versioned_packages=""
 log "Verifying packages..."
 for package in ${packages}; do  
-  if test ! "$(apt-cache show "${package}")"; then
+  if test ! "$(apt-cache show ${package})"; then
     echo "aborted"
     log "Package '${package}' not found." >&2
     exit 5
@@ -74,12 +74,12 @@ set -e
 log "Creating cache key..."
 
 # TODO Can we prove this will happen again?
-normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")"
+normalized_versioned_packages="$(get_normalized_package_list "${versioned_packages}")"
 log "- Normalized package list is '${normalized_versioned_packages}'."
 
 # Forces an update in cases where an accidental breaking change was introduced
 # and a global cache reset is required.
-force_update_inc="0"
+force_update_inc="1"
 
 value="${normalized_versioned_packages} @ ${version} ${force_update_inc}"
 log "- Value to hash is '${value}'."

restore_pkgs.sh

@@ -40,22 +40,22 @@ log "done"
 log_empty_line
 
 # Only search for archived results. Manifest and cache key also live here.
-cached_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar | sort)
+cached_filepaths=$(ls -1 "${cache_dir}"/*.tar | sort)
-cached_pkg_filecount=$(echo ${cached_pkg_filepaths} | wc -w)
+cached_filecount=$(echo ${cached_filepaths} | wc -w)
 
-log "Restoring ${cached_pkg_filecount} packages from cache..."
+log "Restoring ${cached_filecount} packages from cache..."
-for cached_pkg_filepath in ${cached_pkg_filepaths}; do
+for cached_filepath in ${cached_filepaths}; do
 
-  log "- $(basename "${cached_pkg_filepath}") restoring..."
+  log "- $(basename "${cached_filepath}") restoring..."
-  sudo tar -xf "${cached_pkg_filepath}" -C "${cache_restore_root}" > /dev/null
+  sudo tar -xf "${cached_filepath}" -C "${cache_restore_root}" > /dev/null
   log "  done"
 
   # Execute install scripts if available.    
   if test ${execute_install_scripts} == "true"; then
     # May have to add more handling for extracting pre-install script before extracting all files.
     # Keeping it simple for now.
-    execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" preinst install
+    execute_install_script "${cache_restore_root}" "${cached_filepath}" preinst install
-    execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" postinst configure
+    execute_install_script "${cache_restore_root}" "${cached_filepath}" postinst configure
   fi
 done
 log "done"