Optimize installs with apt-fast and various minor cleanups. (#35)
* Fix cut regression. Originally fixed in #17. This was reintroduced when master was sync'd to staging. * Update pre_cache_action.sh * Switch to CLI safe apt command. Address concern in issue #23. * Optimize installs with apt-fast and cleanup logging.
This commit is contained in:
Andrew Walsh
GitHub
parent
97133735cc
commit
c5df606b25
5 changed files with 88 additions and 50 deletions
|
|
@ -7,6 +7,9 @@ set -e
|
|||
script_dir="$(dirname -- "$(realpath -- "${0}")")"
|
||||
source "${script_dir}/lib.sh"
|
||||
|
||||
# Install apt-fast for optimized installs.
|
||||
/bin/bash -c "$(curl -sL https://git.io/vokNn)"
|
||||
|
||||
# Directory that holds the cached packages.
|
||||
cache_dir="${1}"
|
||||
|
||||
|
|
@ -18,28 +21,40 @@ normalized_packages="$(normalize_package_list "${input_packages}")"
|
|||
|
||||
package_count=$(wc -w <<< "${normalized_packages}")
|
||||
log "Clean installing and caching ${package_count} package(s)."
|
||||
|
||||
log_empty_line
|
||||
|
||||
log "Package list:"
|
||||
for package in ${normalized_packages}; do
|
||||
log "- ${package}"
|
||||
done
|
||||
|
||||
log_empty_line
|
||||
|
||||
log "Updating APT package list..."
|
||||
sudo apt-get update > /dev/null
|
||||
echo "done."
|
||||
sudo apt-fast update > /dev/null
|
||||
log "done"
|
||||
|
||||
log_empty_line
|
||||
|
||||
# Strictly contains the requested packages.
|
||||
manifest_main=""
|
||||
# Contains all packages including dependencies.
|
||||
manifest_all=""
|
||||
|
||||
log "Clean installing and caching ${package_count} packages..."
|
||||
log "Gathering install information for ${package_count} packages..."
|
||||
log_empty_line
|
||||
cached_packages=""
|
||||
for package in ${normalized_packages}; do
|
||||
read package_name package_ver < <(get_package_name_ver "${package}")
|
||||
|
||||
# Comma delimited name:ver pairs in the main requested packages manifest.
|
||||
manifest_main="${manifest_main}${package_name}:${package_ver},"
|
||||
|
||||
cached_packages="${cached_packages} ${package_name}:${package_version}"
|
||||
read dep_packages < <(get_dep_packages "${package_name}")
|
||||
cached_packages="${cached_packages} $(echo ${dep_packages} | tr '\n' ' ')"
|
||||
|
||||
if test -z "${dep_packages}"; then
|
||||
dep_packages_text="none";
|
||||
else
|
||||
|
|
@ -49,19 +64,29 @@ for package in ${normalized_packages}; do
|
|||
log "- ${package_name}"
|
||||
log " * Version: ${package_ver}"
|
||||
log " * Dependencies: ${dep_packages_text}"
|
||||
log " * Installing..."
|
||||
# Zero interaction while installing or upgrading the system via apt.
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package_name}" > /dev/null
|
||||
echo "done."
|
||||
log_empty_line
|
||||
done
|
||||
log "done"
|
||||
|
||||
for cache_package in ${package_name}:${package_ver} ${dep_packages}; do
|
||||
cache_filepath="${cache_dir}/${cache_package}.tar.gz"
|
||||
log_empty_line
|
||||
|
||||
log "Clean installing ${package_count} packages..."
|
||||
# Zero interaction while installing or upgrading the system via apt.
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${normalized_packages} > /dev/null
|
||||
log "done"
|
||||
|
||||
log_empty_line
|
||||
|
||||
cached_package_count=$(wc -w <<< "${cached_packages}")
|
||||
log "Caching ${cached_package_count} installed packages..."
|
||||
for cached_package in ${cached_packages}; do
|
||||
cache_filepath="${cache_dir}/${cached_package}.tar.gz"
|
||||
|
||||
if test ! -f "${cache_filepath}"; then
|
||||
read cache_package_name cache_package_ver < <(get_package_name_ver "${cache_package}")
|
||||
log " * Caching ${cache_package_name} to ${cache_filepath}..."
|
||||
read cached_package_name cached_package_ver < <(get_package_name_ver "${cached_package}")
|
||||
log " * Caching ${cached_package_name} to ${cache_filepath}..."
|
||||
# Pipe all package files (no folders) to Tar.
|
||||
dpkg -L "${cache_package_name}" |
|
||||
dpkg -L "${cached_package_name}" |
|
||||
while IFS= read -r f; do
|
||||
if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows
|
||||
done |
|
||||
|
|
@ -70,10 +95,11 @@ for package in ${normalized_packages}; do
|
|||
fi
|
||||
|
||||
# Comma delimited name:ver pairs in the all packages manifest.
|
||||
manifest_all="${manifest_all}${cache_package_name}:${cache_package_ver},"
|
||||
manifest_all="${manifest_all}${cached_package_name}:${cached_package_ver},"
|
||||
done
|
||||
done
|
||||
log "done."
|
||||
log "done (total cache size $(du -h ${cache_dir} | tail -1 | awk '{print $1}'))"
|
||||
|
||||
log_empty_line
|
||||
|
||||
write_manifest "all" "${manifest_all}" "${cache_dir}/manifest_all.log"
|
||||
write_manifest "main" "${manifest_main}" "${cache_dir}/manifest_main.log"
|
||||
|
|
|
|||
13
lib.sh
13
lib.sh
|
|
@ -9,10 +9,10 @@ function normalize_package_list {
|
|||
echo "${sorted}"
|
||||
}
|
||||
|
||||
# Gets a package list of dependencies as common delimited pairs
|
||||
# <name>:<version>,<name:version>...
|
||||
# Gets a package list of dependencies as newline delimited pairs
|
||||
# <name>:<version>\n<name:version>...
|
||||
function get_dep_packages {
|
||||
echo $(apt-get install --dry-run --yes "${1}" | \
|
||||
echo $(apt-fast install --dry-run --yes "${1}" | \
|
||||
grep "^Inst" | sort | awk '{print $2 $3}' | \
|
||||
tr '(' ':' | grep -v "${1}:")
|
||||
}
|
||||
|
|
@ -22,16 +22,19 @@ function get_package_name_ver {
|
|||
IFS=\: read name ver <<< "${1}"
|
||||
# If version not found in the fully qualified package value.
|
||||
if test -z "${ver}"; then
|
||||
ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')"
|
||||
ver="$(grep "Version:" <<< "$(apt-cache show ${name})" | awk '{print $2}')"
|
||||
fi
|
||||
echo "${name}" "${ver}"
|
||||
}
|
||||
|
||||
function log { echo "$(date +%H:%M:%S)" "${@}"; }
|
||||
|
||||
function log_empty_line { echo ""; }
|
||||
|
||||
# Writes the manifest to a specified file.
|
||||
function write_manifest {
|
||||
log "Writing ${1} packages manifest to ${3}..."
|
||||
# 0:-1 to remove trailing comma, delimit by newline and sort
|
||||
echo "${2:0:-1}" | tr ',' '\n' | sort > ${3}
|
||||
log "done."
|
||||
log "done"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,4 +27,5 @@ if [ "$cache_hit" == true ]; then
|
|||
else
|
||||
${script_dir}/install_and_cache_pkgs.sh ~/cache-apt-pkgs ${packages}
|
||||
fi
|
||||
echo ""
|
||||
|
||||
log_empty_line
|
||||
|
|
|
|||
|
|
@ -21,31 +21,35 @@ mkdir -p ${cache_dir}
|
|||
|
||||
log -n "Validating action arguments (version='${version}', packages='${packages}')...";
|
||||
if grep -q " " <<< "${version}"; then
|
||||
log "aborted."
|
||||
log "aborted"
|
||||
log "Version value '${version}' cannot contain spaces." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Is length of string zero?
|
||||
if test -z "${packages}"; then
|
||||
log "aborted."
|
||||
log "aborted"
|
||||
log "Packages argument cannot be empty." >&2
|
||||
exit 2
|
||||
fi
|
||||
log "done."
|
||||
log "done"
|
||||
|
||||
log_empty_line
|
||||
|
||||
versioned_packages=""
|
||||
log -n "Verifying packages..."
|
||||
for package in ${packages}; do
|
||||
if test ! "$(apt show "${package}")"; then
|
||||
echo "aborted."
|
||||
echo "aborted"
|
||||
log "Package '${package}' not found." >&2
|
||||
exit 3
|
||||
fi
|
||||
read package_name package_ver < <(get_package_name_ver "${package}")
|
||||
versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}""
|
||||
done
|
||||
echo "done."
|
||||
echo "done"
|
||||
|
||||
log_empty_line
|
||||
|
||||
# Abort on any failure at this point.
|
||||
set -e
|
||||
|
|
@ -62,7 +66,7 @@ log "- Value to hash is '${value}'."
|
|||
key="$(echo "${value}" | md5sum | cut -f1 -d' ')"
|
||||
log "- Value hashed as '${key}'."
|
||||
|
||||
log "done."
|
||||
log "done"
|
||||
|
||||
key_filepath="${cache_dir}/cache_key.md5"
|
||||
echo ${key} > ${key_filepath}
|
||||
|
|
|
|||
|
|
@ -20,19 +20,23 @@ for cache_filepath in ${cache_filepaths}; do
|
|||
log "- "$(basename ${cache_filepath})""
|
||||
done
|
||||
|
||||
log_empty_line
|
||||
|
||||
log "Reading from main requested packages manifest..."
|
||||
for logline in $(cat "${cache_dir}/manifest_main.log" | tr ',' '\n' ); do
|
||||
log "- $(echo "${logline}" | tr ':' ' ')"
|
||||
done
|
||||
log "done."
|
||||
log "done"
|
||||
|
||||
log_empty_line
|
||||
|
||||
# Only search for archived results. Manifest and cache key also live here.
|
||||
cache_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort)
|
||||
cache_pkg_filecount=$(echo ${cache_pkg_filepaths} | wc -w)
|
||||
log "Restoring ${cache_pkg_filecount} packages from cache..."
|
||||
for cache_pkg_filepath in ${cache_pkg_filepaths}; do
|
||||
log "- $(basename "${cache_pkg_filepath}") restoring..."
|
||||
sudo tar -xf "${cache_pkg_filepath}" -C "${cache_restore_root}" > /dev/null
|
||||
log "done."
|
||||
cached_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort)
|
||||
cached_pkg_filecount=$(echo ${cached_pkg_filepaths} | wc -w)
|
||||
log "Restoring ${cached_pkg_filecount} packages from cache..."
|
||||
for cached_pkg_filepath in ${cached_pkg_filepaths}; do
|
||||
log "- $(basename "${cached_pkg_filepath}") restoring..."
|
||||
sudo tar -xf "${cached_pkg_filepath}" -C "${cache_restore_root}" > /dev/null
|
||||
log " done"
|
||||
done
|
||||
log "done."
|
||||
log "done"
|
||||
|
|
|
|||
Loading…
Reference in a new issue