Optimize installs with apt-fast and various minor cleanups. (#35)

* Fix cut regression.

Originally fixed in #17. This was reintroduced when master was sync'd to staging.

* Update pre_cache_action.sh

* Switch to CLI safe apt command.

Address concern in issue #23.

* Optimize installs with apt-fast and cleanup logging.

install_and_cache_pkgs.sh

@@ -7,6 +7,9 @@ set -e
 script_dir="$(dirname -- "$(realpath -- "${0}")")"
 source "${script_dir}/lib.sh"
 
+# Install apt-fast for optimized installs.
+/bin/bash -c "$(curl -sL https://git.io/vokNn)"
+
 # Directory that holds the cached packages.
 cache_dir="${1}"
 
@@ -18,28 +21,40 @@ normalized_packages="$(normalize_package_list "${input_packages}")"
 
 package_count=$(wc -w <<< "${normalized_packages}")
 log "Clean installing and caching ${package_count} package(s)."
+
+log_empty_line
+
 log "Package list:"
 for package in ${normalized_packages}; do
   log "- ${package}"
 done
 
+log_empty_line
+
 log "Updating APT package list..."
-sudo apt-get update > /dev/null
+sudo apt-fast update > /dev/null
-echo "done."
+log "done"
+
+log_empty_line
 
 # Strictly contains the requested packages.
 manifest_main=""
 # Contains all packages including dependencies.
 manifest_all=""
 
-log "Clean installing and caching ${package_count} packages..."
+log "Gathering install information for ${package_count} packages..."
+log_empty_line
+cached_packages=""
 for package in ${normalized_packages}; do
   read package_name package_ver < <(get_package_name_ver "${package}")    
 
   # Comma delimited name:ver pairs in the main requested packages manifest.
   manifest_main="${manifest_main}${package_name}:${package_ver},"
 
+  cached_packages="${cached_packages} ${package_name}:${package_version}"
   read dep_packages < <(get_dep_packages "${package_name}")
+  cached_packages="${cached_packages} $(echo ${dep_packages} | tr '\n' ' ')"
+
   if test -z "${dep_packages}"; then
     dep_packages_text="none";
   else
@@ -49,31 +64,42 @@ for package in ${normalized_packages}; do
   log "- ${package_name}"
   log "  * Version: ${package_ver}"
   log "  * Dependencies: ${dep_packages_text}"
-  log "  * Installing..."
+  log_empty_line
-  # Zero interaction while installing or upgrading the system via apt.
-  sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package_name}" > /dev/null
-  echo "done."
-
-  for cache_package in ${package_name}:${package_ver} ${dep_packages}; do
-    cache_filepath="${cache_dir}/${cache_package}.tar.gz"
-
-    if test ! -f "${cache_filepath}"; then
-      read cache_package_name cache_package_ver < <(get_package_name_ver "${cache_package}")
-      log "  * Caching ${cache_package_name} to ${cache_filepath}..."
-      # Pipe all package files (no folders) to Tar.
-      dpkg -L "${cache_package_name}" |
-        while IFS= read -r f; do     
-          if test -f $f || test -L $f; then echo "${f:1}"; fi;  #${f:1} removes the leading slash that Tar disallows
-        done | 
-        xargs tar -czf "${cache_filepath}" -C /      
-      log "done (compressed size $(du -h "${cache_filepath}" | cut -f1))."
-    fi
-
-    # Comma delimited name:ver pairs in the all packages manifest.
-    manifest_all="${manifest_all}${cache_package_name}:${cache_package_ver},"
-  done  
 done
-log "done."
+log "done"
+
+log_empty_line
+
+log "Clean installing ${package_count} packages..."
+# Zero interaction while installing or upgrading the system via apt.
+sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${normalized_packages} > /dev/null
+log "done"
+
+log_empty_line
+
+cached_package_count=$(wc -w <<< "${cached_packages}")
+log "Caching ${cached_package_count} installed packages..."
+for cached_package in ${cached_packages}; do
+  cache_filepath="${cache_dir}/${cached_package}.tar.gz"
+
+  if test ! -f "${cache_filepath}"; then
+    read cached_package_name cached_package_ver < <(get_package_name_ver "${cached_package}")
+    log "  * Caching ${cached_package_name} to ${cache_filepath}..."
+    # Pipe all package files (no folders) to Tar.
+    dpkg -L "${cached_package_name}" |
+      while IFS= read -r f; do     
+        if test -f $f || test -L $f; then echo "${f:1}"; fi;  #${f:1} removes the leading slash that Tar disallows
+      done |
+      xargs tar -czf "${cache_filepath}" -C /      
+    log "    done (compressed size $(du -h "${cache_filepath}" | cut -f1))."
+  fi
+
+  # Comma delimited name:ver pairs in the all packages manifest.
+  manifest_all="${manifest_all}${cached_package_name}:${cached_package_ver},"
+done
+log "done (total cache size $(du -h ${cache_dir} | tail -1 | awk '{print $1}'))"
+
+log_empty_line
 
 write_manifest "all" "${manifest_all}" "${cache_dir}/manifest_all.log"
 write_manifest "main" "${manifest_main}" "${cache_dir}/manifest_main.log"

lib.sh

@@ -9,10 +9,10 @@ function normalize_package_list {
   echo "${sorted}"  
 }
 
-# Gets a package list of dependencies as common delimited pairs
+# Gets a package list of dependencies as newline delimited pairs
-#   <name>:<version>,<name:version>...
+#   <name>:<version>\n<name:version>...
 function get_dep_packages {  
-  echo $(apt-get install --dry-run --yes "${1}" | \
+  echo $(apt-fast install --dry-run --yes "${1}" | \
     grep "^Inst" | sort | awk '{print $2 $3}' | \
     tr '(' ':' | grep -v "${1}:")
 }
@@ -22,16 +22,19 @@ function get_package_name_ver {
   IFS=\: read name ver <<< "${1}"
   # If version not found in the fully qualified package value.
   if test -z "${ver}"; then
-    ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')"
+    ver="$(grep "Version:" <<< "$(apt-cache show ${name})" | awk '{print $2}')"
   fi
   echo "${name}" "${ver}"
 }
 
 function log { echo "$(date +%H:%M:%S)" "${@}"; }
 
+function log_empty_line { echo ""; }
+
+# Writes the manifest to a specified file.
 function write_manifest {
   log "Writing ${1} packages manifest to ${3}..."  
   # 0:-1 to remove trailing comma, delimit by newline and sort
   echo "${2:0:-1}" | tr ',' '\n' | sort > ${3}
-  log "done."
+  log "done"
 }

post_cache_action.sh

@@ -27,4 +27,5 @@ if [ "$cache_hit" == true ]; then
 else
   ${script_dir}/install_and_cache_pkgs.sh ~/cache-apt-pkgs ${packages}
 fi
-echo ""
+
+log_empty_line

pre_cache_action.sh

@@ -21,31 +21,35 @@ mkdir -p ${cache_dir}
 
 log -n "Validating action arguments (version='${version}', packages='${packages}')...";
 if grep -q " " <<< "${version}"; then
-  log "aborted." 
+  log "aborted" 
   log "Version value '${version}' cannot contain spaces." >&2
   exit 1
 fi
 
 # Is length of string zero?
 if test -z "${packages}"; then
-  log "aborted."
+  log "aborted"
   log "Packages argument cannot be empty." >&2
   exit 2
 fi
-log "done."
+log "done"
+
+log_empty_line
 
 versioned_packages=""
 log -n "Verifying packages..."
 for package in ${packages}; do 
   if test ! "$(apt show "${package}")"; then
-    echo "aborted."
+    echo "aborted"
     log "Package '${package}' not found." >&2
     exit 3
   fi
   read package_name package_ver < <(get_package_name_ver "${package}")
   versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}""
 done
-echo "done."
+echo "done"
+
+log_empty_line
 
 # Abort on any failure at this point.
 set -e
@@ -62,7 +66,7 @@ log "- Value to hash is '${value}'."
 key="$(echo "${value}" | md5sum | cut -f1 -d' ')"
 log "- Value hashed as '${key}'."
 
-log "done."
+log "done"
 
 key_filepath="${cache_dir}/cache_key.md5"
 echo ${key} > ${key_filepath}

restore_pkgs.sh

@@ -20,19 +20,23 @@ for cache_filepath in ${cache_filepaths}; do
   log "- "$(basename ${cache_filepath})""
 done
 
+log_empty_line
+
 log "Reading from main requested packages manifest..."
 for logline in $(cat "${cache_dir}/manifest_main.log" | tr ',' '\n' ); do
   log "- $(echo "${logline}" | tr ':' ' ')"
 done
-log "done."
+log "done"
+
+log_empty_line
 
 # Only search for archived results. Manifest and cache key also live here.
-cache_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort)
+cached_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort)
-cache_pkg_filecount=$(echo ${cache_pkg_filepaths} | wc -w)
+cached_pkg_filecount=$(echo ${cached_pkg_filepaths} | wc -w)
-log "Restoring ${cache_pkg_filecount} packages from cache..."
+log "Restoring ${cached_pkg_filecount} packages from cache..."
-for cache_pkg_filepath in ${cache_pkg_filepaths}; do
+for cached_pkg_filepath in ${cached_pkg_filepaths}; do
-  log "- $(basename "${cache_pkg_filepath}") restoring..."
+  log "- $(basename "${cached_pkg_filepath}") restoring..."
-  sudo tar -xf "${cache_pkg_filepath}" -C "${cache_restore_root}" > /dev/null
+  sudo tar -xf "${cached_pkg_filepath}" -C "${cache_restore_root}" > /dev/null
-  log "done."
+  log "  done"
 done
-log "done."
+log "done"