Merge branch 'dev' into master
This commit is contained in:
Andrew Walsh
GitHub
commit
135ee20306
5 changed files with 77 additions and 83 deletions
|
|
@ -6,7 +6,7 @@ set -e
|
|||
# Debug mode for diagnosing issues.
|
||||
# Setup first before other operations.
|
||||
debug="${2}"
|
||||
test ${debug} == "true" && set -x
|
||||
test "${debug}" = "true" && set -x
|
||||
|
||||
# Include library.
|
||||
script_dir="$(dirname -- "$(realpath -- "${0}")")"
|
||||
|
|
@ -18,20 +18,23 @@ cache_dir="${1}"
|
|||
# List of the packages to use.
|
||||
input_packages="${@:3}"
|
||||
|
||||
# Trim commas, excess spaces, and sort.
|
||||
normalized_packages="$(normalize_package_list "${input_packages}")"
|
||||
# Trim commas, excess spaces, sort, and version syntax.
|
||||
#
|
||||
# NOTE: Unless specified, all APT package listings of name and version use
|
||||
# colon delimited and not equals delimited syntax (i.e. <name>[:=]<ver>).
|
||||
packages="$(get_normalized_package_list "${input_packages}")"
|
||||
|
||||
package_count=$(wc -w <<< "${normalized_packages}")
|
||||
package_count=$(wc -w <<< "${packages}")
|
||||
log "Clean installing and caching ${package_count} package(s)."
|
||||
|
||||
log_empty_line
|
||||
|
||||
manifest_main=""
|
||||
log "Package list:"
|
||||
for package in ${normalized_packages}; do
|
||||
for package in ${packages}; do
|
||||
read package_name package_ver < <(get_package_name_ver "${package}")
|
||||
manifest_main="${manifest_main}${package_name}:${package_ver},"
|
||||
log "- ${package_name}:${package_ver}"
|
||||
manifest_main="${manifest_main}${package_name}=${package_ver},"
|
||||
log "- ${package_name} (${package_ver})"
|
||||
done
|
||||
write_manifest "main" "${manifest_main}" "${cache_dir}/manifest_main.log"
|
||||
|
||||
|
|
@ -65,7 +68,7 @@ install_log_filepath="${cache_dir}/install.log"
|
|||
|
||||
log "Clean installing ${package_count} packages..."
|
||||
# Zero interaction while installing or upgrading the system via apt.
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${normalized_packages} > "${install_log_filepath}"
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${packages} > "${install_log_filepath}"
|
||||
log "done"
|
||||
log "Installation log written to ${install_log_filepath}"
|
||||
|
||||
|
|
@ -74,34 +77,35 @@ log_empty_line
|
|||
installed_packages=$(get_installed_packages "${install_log_filepath}")
|
||||
log "Installed package list:"
|
||||
for installed_package in ${installed_packages}; do
|
||||
log "- ${installed_package}"
|
||||
# Reformat for human friendly reading.
|
||||
log "- $(echo ${installed_package} | awk -F\= '{print $1" ("$2")"}')"
|
||||
done
|
||||
|
||||
log_empty_line
|
||||
|
||||
installed_package_count=$(wc -w <<< "${installed_packages}")
|
||||
log "Caching ${installed_package_count} installed packages..."
|
||||
installed_packages_count=$(wc -w <<< "${installed_packages}")
|
||||
log "Caching ${installed_packages_count} installed packages..."
|
||||
for installed_package in ${installed_packages}; do
|
||||
cache_filepath="${cache_dir}/${installed_package}.tar"
|
||||
|
||||
# Sanity test in case APT enumerates duplicates.
|
||||
if test ! -f "${cache_filepath}"; then
|
||||
read installed_package_name installed_package_ver < <(get_package_name_ver "${installed_package}")
|
||||
log " * Caching ${installed_package_name} to ${cache_filepath}..."
|
||||
read package_name package_ver < <(get_package_name_ver "${installed_package}")
|
||||
log " * Caching ${package_name} to ${cache_filepath}..."
|
||||
|
||||
# Pipe all package files (no folders) and installation control data to Tar.
|
||||
{ dpkg -L "${installed_package_name}" \
|
||||
& get_install_filepath "" "${package_name}" "preinst" \
|
||||
& get_install_filepath "" "${package_name}" "postinst"; } |
|
||||
{ dpkg -L "${package_name}" \
|
||||
& get_install_script_filepath "" "${package_name}" "preinst" \
|
||||
& get_install_script_filepath "" "${package_name}" "postinst"; } |
|
||||
while IFS= read -r f; do test -f "${f}" -o -L "${f}" && get_tar_relpath "${f}"; done |
|
||||
xargs -I {} echo \"{}\" |
|
||||
xargs -I {} echo \'{}\' | # Single quotes ensure literals like backslash get captured.
|
||||
sudo xargs tar -cf "${cache_filepath}" -C /
|
||||
|
||||
log " done (compressed size $(du -h "${cache_filepath}" | cut -f1))."
|
||||
fi
|
||||
|
||||
# Comma delimited name:ver pairs in the all packages manifest.
|
||||
manifest_all="${manifest_all}${installed_package_name}:${installed_package_ver},"
|
||||
manifest_all="${manifest_all}${package_name}=${package_ver},"
|
||||
done
|
||||
log "done (total cache size $(du -h ${cache_dir} | tail -1 | awk '{print $1}'))"
|
||||
|
||||
|
|
|
|||
90
lib.sh
90
lib.sh
|
|
@ -11,9 +11,9 @@
|
|||
# Filepath of the install script, otherwise an empty string.
|
||||
###############################################################################
|
||||
function execute_install_script {
|
||||
local package_name=$(basename ${2} | awk -F\: '{print $1}')
|
||||
local package_name=$(basename ${2} | awk -F\= '{print $1}')
|
||||
local install_script_filepath=$(\
|
||||
get_install_filepath "${1}" "${package_name}" "${3}")
|
||||
get_install_script_filepath "${1}" "${package_name}" "${3}")
|
||||
if test ! -z "${install_script_filepath}"; then
|
||||
log "- Executing ${install_script_filepath}..."
|
||||
# Don't abort on errors; dpkg-trigger will error normally since it is
|
||||
|
|
@ -23,21 +23,39 @@ function execute_install_script {
|
|||
fi
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Gets the Debian install script filepath.
|
||||
# Arguments:
|
||||
# Root directory to search from.
|
||||
# Name of the unqualified package to search for.
|
||||
# Extension of the installation script (preinst, postinst)
|
||||
# Returns:
|
||||
# Filepath of the script file, otherwise an empty string.
|
||||
###############################################################################
|
||||
function get_install_script_filepath {
|
||||
# Filename includes arch (e.g. amd64).
|
||||
local filepath="$(\
|
||||
ls -1 ${1}var/lib/dpkg/info/${2}*.${3} 2> /dev/null \
|
||||
| grep -E ${2}'(:.*)?.'${3} | head -1 || true)"
|
||||
test "${filepath}" && echo "${filepath}"
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Gets a list of installed packages from a Debian package installation log.
|
||||
# Arguments:
|
||||
# The filepath of the Debian install log.
|
||||
# Returns:
|
||||
# The list of space delimited pairs with each pair colon delimited.
|
||||
# <name>:<version> <name:version>...
|
||||
# The list of colon delimited action syntax pairs with each pair equals
|
||||
# delimited. <name>:<version> <name>:<version>...
|
||||
###############################################################################
|
||||
function get_installed_packages {
|
||||
local install_log_filepath="${1}"
|
||||
local regex="^Unpacking ([^ :]+)([^ ]+)? (\[[^ ]+\]\s)?\(([^ )]+)"
|
||||
local dep_packages=""
|
||||
while read -r line; do
|
||||
# ${regex} should be unquoted since it isn't a literal.
|
||||
if [[ "${line}" =~ ${regex} ]]; then
|
||||
dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[4]} "
|
||||
dep_packages="${dep_packages}${BASH_REMATCH[1]}=${BASH_REMATCH[4]} "
|
||||
else
|
||||
log_err "Unable to parse package name and version from \"${line}\""
|
||||
exit 2
|
||||
|
|
@ -51,48 +69,38 @@ function get_installed_packages {
|
|||
}
|
||||
|
||||
###############################################################################
|
||||
# Splits a fully qualified package into name and version.
|
||||
# Splits a fully action syntax APT package into the name and version.
|
||||
# Arguments:
|
||||
# The colon delimited package pair or just the package name.
|
||||
# The action syntax colon delimited package pair or just the package name.
|
||||
# Returns:
|
||||
# The package name and version pair.
|
||||
###############################################################################
|
||||
function get_package_name_ver {
|
||||
IFS=\: read name ver <<< "${1}"
|
||||
local ORIG_IFS="${IFS}"
|
||||
IFS=\= read name ver <<< "${1}"
|
||||
IFS="${ORIG_IFS}"
|
||||
# If version not found in the fully qualified package value.
|
||||
if test -z "${ver}"; then
|
||||
ver="$(grep "Version:" <<< "$(apt-cache show ${name})" | awk '{print $2}')"
|
||||
fi
|
||||
echo "${name}" "${ver}"
|
||||
echo "${name}" "${ver}"
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Gets the package name from the cached package filepath in the
|
||||
# path/to/cache/dir/<name>:<version>.tar format.
|
||||
# Sorts given packages by name and split on commas and/or spaces.
|
||||
# Arguments:
|
||||
# Filepath to the cached packaged.
|
||||
# The comma and/or space delimited list of packages.
|
||||
# Returns:
|
||||
# The package name.
|
||||
# Sorted list of space delimited packages.
|
||||
###############################################################################
|
||||
function get_package_name_from_cached_filepath {
|
||||
basename ${cached_pkg_filepath} | awk -F\: '{print $1}'
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Gets the Debian install script file location.
|
||||
# Arguments:
|
||||
# Root directory to search from.
|
||||
# Name of the unqualified package to search for.
|
||||
# Extension of the installation script (preinst, postinst)
|
||||
# Returns:
|
||||
# Filepath of the script file, otherwise an empty string.
|
||||
###############################################################################
|
||||
function get_install_filepath {
|
||||
# Filename includes arch (e.g. amd64).
|
||||
local filepath="$(\
|
||||
ls -1 ${1}var/lib/dpkg/info/${2}*.${3} 2> /dev/null \
|
||||
| grep -E ${2}'(:.*)?.'${3} | head -1 || true)"
|
||||
test "${filepath}" && echo "${filepath}"
|
||||
function get_normalized_package_list {
|
||||
# Remove commas, and block scalar folded backslashes.
|
||||
local stripped=$(echo "${1}" | sed 's/[,\]/ /g')
|
||||
# Remove extraneous spaces at the middle, beginning, and end.
|
||||
local trimmed="$(\
|
||||
echo "${stripped}" \
|
||||
| sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')"
|
||||
echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' '
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
|
|
@ -117,24 +125,6 @@ function log_err { >&2 echo "$(date +%H:%M:%S)" "${@}"; }
|
|||
|
||||
function log_empty_line { echo ""; }
|
||||
|
||||
###############################################################################
|
||||
# Sorts given packages by name and split on commas and/or spaces.
|
||||
# Arguments:
|
||||
# The comma and/or space delimited list of packages.
|
||||
# Returns:
|
||||
# Sorted list of space delimited packages.
|
||||
###############################################################################
|
||||
function normalize_package_list {
|
||||
# Remove commas, and block scalar folded backslashes.
|
||||
local stripped=$(echo "${1}" | sed 's/[,\]/ /g')
|
||||
# Remove extraneous spaces at the middle, beginning, and end.
|
||||
local trimmed="$(\
|
||||
echo "${stripped}" \
|
||||
| sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')"
|
||||
local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')"
|
||||
echo "${sorted}"
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Validates an argument to be of a boolean value.
|
||||
# Arguments:
|
||||
|
|
|
|||
|
|
@ -23,12 +23,12 @@ execute_install_scripts="${4}"
|
|||
|
||||
# Debug mode for diagnosing issues.
|
||||
debug="${5}"
|
||||
test ${debug} == "true" && set -x
|
||||
test "${debug}" = "true" && set -x
|
||||
|
||||
# List of the packages to use.
|
||||
packages="${@:6}"
|
||||
|
||||
if [ "$cache_hit" == true ]; then
|
||||
if test "${cache_hit}" = "true"; then
|
||||
${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}"
|
||||
else
|
||||
${script_dir}/install_and_cache_pkgs.sh "${cache_dir}" "${debug}" ${packages}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ debug="${4}"
|
|||
input_packages="${@:5}"
|
||||
|
||||
# Trim commas, excess spaces, and sort.
|
||||
packages="$(normalize_package_list "${input_packages}")"
|
||||
packages="$(get_normalized_package_list "${input_packages}")"
|
||||
|
||||
# Create cache directory so artifacts can be saved.
|
||||
mkdir -p ${cache_dir}
|
||||
|
|
@ -55,8 +55,8 @@ log_empty_line
|
|||
|
||||
versioned_packages=""
|
||||
log "Verifying packages..."
|
||||
for package in ${packages}; do
|
||||
if test ! "$(apt-cache show "${package}")"; then
|
||||
for package in ${packages}; do
|
||||
if test ! "$(apt-cache show ${package})"; then
|
||||
echo "aborted"
|
||||
log "Package '${package}' not found." >&2
|
||||
exit 5
|
||||
|
|
@ -74,12 +74,12 @@ set -e
|
|||
log "Creating cache key..."
|
||||
|
||||
# TODO Can we prove this will happen again?
|
||||
normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")"
|
||||
normalized_versioned_packages="$(get_normalized_package_list "${versioned_packages}")"
|
||||
log "- Normalized package list is '${normalized_versioned_packages}'."
|
||||
|
||||
# Forces an update in cases where an accidental breaking change was introduced
|
||||
# and a global cache reset is required.
|
||||
force_update_inc="0"
|
||||
force_update_inc="1"
|
||||
|
||||
value="${normalized_versioned_packages} @ ${version} ${force_update_inc}"
|
||||
log "- Value to hash is '${value}'."
|
||||
|
|
|
|||
|
|
@ -40,22 +40,22 @@ log "done"
|
|||
log_empty_line
|
||||
|
||||
# Only search for archived results. Manifest and cache key also live here.
|
||||
cached_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar | sort)
|
||||
cached_pkg_filecount=$(echo ${cached_pkg_filepaths} | wc -w)
|
||||
cached_filepaths=$(ls -1 "${cache_dir}"/*.tar | sort)
|
||||
cached_filecount=$(echo ${cached_filepaths} | wc -w)
|
||||
|
||||
log "Restoring ${cached_pkg_filecount} packages from cache..."
|
||||
for cached_pkg_filepath in ${cached_pkg_filepaths}; do
|
||||
log "Restoring ${cached_filecount} packages from cache..."
|
||||
for cached_filepath in ${cached_filepaths}; do
|
||||
|
||||
log "- $(basename "${cached_pkg_filepath}") restoring..."
|
||||
sudo tar -xf "${cached_pkg_filepath}" -C "${cache_restore_root}" > /dev/null
|
||||
log "- $(basename "${cached_filepath}") restoring..."
|
||||
sudo tar -xf "${cached_filepath}" -C "${cache_restore_root}" > /dev/null
|
||||
log " done"
|
||||
|
||||
# Execute install scripts if available.
|
||||
if test ${execute_install_scripts} == "true"; then
|
||||
# May have to add more handling for extracting pre-install script before extracting all files.
|
||||
# Keeping it simple for now.
|
||||
execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" preinst install
|
||||
execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" postinst configure
|
||||
execute_install_script "${cache_restore_root}" "${cached_filepath}" preinst install
|
||||
execute_install_script "${cache_restore_root}" "${cached_filepath}" postinst configure
|
||||
fi
|
||||
done
|
||||
log "done"
|
||||
|
|
|
|||
Loading…
Reference in a new issue