Copy from staging to dev.
This commit is contained in:
awalsh128
parent
4d55f8a4fb
commit
b61cf9a6f1
9 changed files with 154 additions and 97 deletions
19
.github/workflows/pub_dev_push_event.yml
vendored
Normal file
19
.github/workflows/pub_dev_push_event.yml
vendored
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
name: Publish Dev Push Event
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
|
||||
jobs:
|
||||
publish_event:
|
||||
runs-on: ubuntu-latest
|
||||
name: Publish dev push
|
||||
steps:
|
||||
- run: |
|
||||
curl -i \
|
||||
-X POST \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
-H "Authorization: token ${{ secrets.PUBLISH_PUSH_TOKEN }}" \
|
||||
https://api.github.com/repos/awalsh128/cache-apt-pkgs-action-ci/dispatches \
|
||||
-d '{"event_type":"dev_push"}'
|
||||
2
.github/workflows/pub_master_push_event.yml
vendored
2
.github/workflows/pub_master_push_event.yml
vendored
|
|
@ -8,7 +8,7 @@ on:
|
|||
jobs:
|
||||
publish_event:
|
||||
runs-on: ubuntu-latest
|
||||
name: Publish staging push
|
||||
name: Publish master push
|
||||
steps:
|
||||
- run: |
|
||||
curl -i \
|
||||
|
|
|
|||
|
|
@ -24,8 +24,8 @@ Create a workflow `.yml` file in your repositories `.github/workflows` directory
|
|||
### Outputs
|
||||
|
||||
* `cache-hit` - A boolean value to indicate a cache was found for the packages requested.
|
||||
* `package-version-list` - The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. \<package1>:<version1\>,\<package2>:\<version2>,...).
|
||||
|
||||
* `package-version-list` - The main requested packages and versions that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. \<package1>:<version1\>,\<package2>:\<version2>,...).
|
||||
* `all-package-version-list` - All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. \<package1>:<version1\>,\<package2>:\<version2>,...).
|
||||
|
||||
### Cache scopes
|
||||
|
||||
|
|
|
|||
11
action.yml
11
action.yml
|
|
@ -26,7 +26,10 @@ outputs:
|
|||
# Need to output true and false instead of true and nothing.
|
||||
value: ${{ steps.load-cache.outputs.cache-hit || false }}
|
||||
package-version-list:
|
||||
description: 'The packages and versions that are installed as a comma delimited list with colon delimit on the package version (i.e. <package>:<version,<package>:<version>).'
|
||||
description: 'The main requested packages and versions that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. <package>:<version,<package>:<version>).'
|
||||
value: ${{ steps.post-cache.outputs.package-version-list }}
|
||||
all-package-version-list:
|
||||
description: 'All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with colon delimit on the package version (i.e. <package>:<version,<package>:<version>).'
|
||||
value: ${{ steps.post-cache.outputs.package-version-list }}
|
||||
|
||||
runs:
|
||||
|
|
@ -42,7 +45,7 @@ runs:
|
|||
shell: bash
|
||||
|
||||
- id: load-cache
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/cache-apt-pkgs
|
||||
key: cache-apt-pkgs_${{ env.CACHE_KEY }}
|
||||
|
|
@ -54,5 +57,7 @@ runs:
|
|||
/ \
|
||||
"${{ steps.load-cache.outputs.cache-hit }}" \
|
||||
${{ inputs.packages }}
|
||||
echo "::set-output name=package-version-list::$(cat ~/cache-apt-pkgs/manifest.log)"
|
||||
function create_list { local list=$(cat ~/cache-apt-pkgs/manifest_${1}.log | tr '\n' ','); echo ${list:0:-1}; };
|
||||
echo "::set-output name=package-version-list::$(create_list main)"
|
||||
echo "::set-output name=all-package-version-list::$(create_list all)"
|
||||
shell: bash
|
||||
|
|
|
|||
|
|
@ -8,61 +8,72 @@ script_dir="$(dirname -- "$(realpath -- "${0}")")"
|
|||
source "${script_dir}/lib.sh"
|
||||
|
||||
# Directory that holds the cached packages.
|
||||
cache_dir=$1
|
||||
cache_dir="${1}"
|
||||
|
||||
# List of the packages to use.
|
||||
input_packages="${@:2}"
|
||||
|
||||
# Trim commas, excess spaces, and sort.
|
||||
packages="$(normalize_package_list "${input_packages}")"
|
||||
normalized_packages="$(normalize_package_list "${input_packages}")"
|
||||
|
||||
package_count=$(echo "${packages}" | wc -w)
|
||||
echo "Clean installing and caching ${package_count} package(s)."
|
||||
echo "Package list:"
|
||||
for package in "${packages}"; do
|
||||
echo "- ${package}"
|
||||
package_count=$(wc -w <<< "${normalized_packages}")
|
||||
log "Clean installing and caching ${package_count} package(s)."
|
||||
log "Package list:"
|
||||
for package in ${normalized_packages}; do
|
||||
log "- ${package}"
|
||||
done
|
||||
|
||||
echo -n "Updating APT package list..."
|
||||
log "Updating APT package list..."
|
||||
sudo apt-get update > /dev/null
|
||||
echo "done."
|
||||
|
||||
manifest=""
|
||||
echo "Clean installing and caching ${package_count} packages..."
|
||||
for package in "${packages}"; do
|
||||
get_package_name_ver "${package}" # -> package_name, package_ver
|
||||
package_deps="$(apt-get install --dry-run --yes "${package_name}" | grep "^Inst" | awk '{print $2}')"
|
||||
# Strictly contains the requested packages.
|
||||
manifest_main=""
|
||||
# Contains all packages including dependencies.
|
||||
manifest_all=""
|
||||
|
||||
echo "- ${package_name}"
|
||||
echo " * Version: ${package_ver}"
|
||||
echo " * Dependencies: ${package_deps}"
|
||||
echo -n " * Installing..."
|
||||
log "Clean installing and caching ${package_count} packages..."
|
||||
for package in ${normalized_packages}; do
|
||||
read package_name package_ver < <(get_package_name_ver "${package}")
|
||||
|
||||
# Comma delimited name:ver pairs in the main requested packages manifest.
|
||||
manifest_main="${manifest_main}${package_name}:${package_ver},"
|
||||
|
||||
read dep_packages < <(get_dep_packages "${package_name}")
|
||||
if test -z "${dep_packages}"; then
|
||||
dep_packages_text="none";
|
||||
else
|
||||
dep_packages_text="${dep_packages}"
|
||||
fi
|
||||
|
||||
log "- ${package_name}"
|
||||
log " * Version: ${package_ver}"
|
||||
log " * Dependencies: ${dep_packages_text}"
|
||||
log " * Installing..."
|
||||
# Zero interaction while installing or upgrading the system via apt.
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package}" > /dev/null
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package_name}" > /dev/null
|
||||
echo "done."
|
||||
|
||||
for cache_package in "${package_deps}"; do
|
||||
for cache_package in ${package_name}:${package_ver} ${dep_packages}; do
|
||||
cache_filepath="${cache_dir}/${cache_package}.tar.gz"
|
||||
|
||||
if test ! -f "${cache_filepath}"; then
|
||||
get_package_name_ver "${cache_package}" # -> package_name, package_ver
|
||||
echo -n " Caching ${package_name} to ${cache_filepath}..."
|
||||
read cache_package_name cache_package_ver < <(get_package_name_ver "${cache_package}")
|
||||
log " * Caching ${cache_package_name} to ${cache_filepath}..."
|
||||
# Pipe all package files (no folders) to Tar.
|
||||
dpkg -L "${package_name}" |
|
||||
dpkg -L "${cache_package_name}" |
|
||||
while IFS= read -r f; do
|
||||
if test -f $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows
|
||||
if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows
|
||||
done |
|
||||
xargs tar -czf "${cache_filepath}" -C /
|
||||
echo "done."
|
||||
# Add package to manifest
|
||||
manifest="${manifest}${package_name}:$(dpkg -s "${package_name}" | grep Version | awk '{print $2}'),"
|
||||
log "done (compressed size $(du -h "${cache_filepath}" | cut -f1))."
|
||||
fi
|
||||
|
||||
# Comma delimited name:ver pairs in the all packages manifest.
|
||||
manifest_all="${manifest_all}${cache_package_name}:${cache_package_ver},"
|
||||
done
|
||||
done
|
||||
echo "done."
|
||||
log "done."
|
||||
|
||||
manifest_filepath="${cache_dir}/manifest.log"
|
||||
echo -n "Writing package manifest to ${manifest_filepath}..."
|
||||
# Remove trailing comma and write to manifest file.
|
||||
echo "${manifest:0:-1}" > "${manifest_filepath}"
|
||||
echo "done."
|
||||
write_manifest "all" "${manifest_all}" "${cache_dir}/manifest_all.log"
|
||||
write_manifest "main" "${manifest_main}" "${cache_dir}/manifest_main.log"
|
||||
|
|
|
|||
30
lib.sh
30
lib.sh
|
|
@ -1,19 +1,37 @@
|
|||
#!/bin/bash -x
|
||||
#!/bin/bash
|
||||
|
||||
# Sort these packages by name and split on commas.
|
||||
function normalize_package_list {
|
||||
stripped="$(echo ${1} | sed 's/,//g')"
|
||||
local stripped=$(echo "${1}" | sed 's/,//g')
|
||||
# Remove extraneous spaces at the middle, beginning, and end.
|
||||
trimmed="$(echo ${stripped} | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')"
|
||||
echo "${trimmed}" | sort
|
||||
local trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')"
|
||||
local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')"
|
||||
echo "${sorted}"
|
||||
}
|
||||
|
||||
# Gets a package list of dependencies as common delimited pairs
|
||||
# <name>:<version>,<name:version>...
|
||||
function get_dep_packages {
|
||||
echo $(apt-get install --dry-run --yes "${1}" | \
|
||||
grep "^Inst" | sort | awk '{print $2 $3}' | \
|
||||
tr '(' ':' | grep -v "${1}:")
|
||||
}
|
||||
|
||||
# Split fully qualified package into name and version
|
||||
function get_package_name_ver {
|
||||
IFS=\= read name ver <<< "${1}"
|
||||
IFS=\: read name ver <<< "${1}"
|
||||
# If version not found in the fully qualified package value.
|
||||
if test -z "${ver}"; then
|
||||
ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')"
|
||||
fi
|
||||
echo 'package_name="${name}"; package_ver="${ver}"'
|
||||
echo "${name}" "${ver}"
|
||||
}
|
||||
|
||||
function log { echo "$(date +%H:%M:%S)" "${@}"; }
|
||||
|
||||
function write_manifest {
|
||||
log "Writing ${1} packages manifest to ${3}..."
|
||||
# 0:-1 to remove trailing comma, delimit by newline and sort
|
||||
echo "${2:0:-1}" | tr ',' '\n' | sort > ${3}
|
||||
log "done."
|
||||
}
|
||||
|
|
@ -3,24 +3,28 @@
|
|||
# Fail on any error.
|
||||
set -e
|
||||
|
||||
# Include library.
|
||||
script_dir="$(dirname -- "$(realpath -- "${0}")")"
|
||||
source "${script_dir}/lib.sh"
|
||||
|
||||
# Directory that holds the cached packages.
|
||||
cache_dir=$1
|
||||
cache_dir="${1}"
|
||||
|
||||
# Root directory to untar the cached packages to.
|
||||
# Typically filesystem root '/' but can be changed for testing.
|
||||
cache_restore_root=$2
|
||||
cache_restore_root="${2}"
|
||||
|
||||
# Indicates that the cache was found.
|
||||
cache_hit=$3
|
||||
cache_hit="${3}"
|
||||
|
||||
# List of the packages to use.
|
||||
packages="${@:4}"
|
||||
|
||||
script_dir=$(dirname $0)
|
||||
script_dir="$(dirname -- "$(realpath -- "${0}")")"
|
||||
|
||||
if [ "$cache_hit" == true ]; then
|
||||
$script_dir/restore_pkgs.sh ~/cache-apt-pkgs $cache_restore_root
|
||||
${script_dir}/restore_pkgs.sh ~/cache-apt-pkgs "${cache_restore_root}"
|
||||
else
|
||||
$script_dir/install_and_cache_pkgs.sh ~/cache-apt-pkgs $packages
|
||||
${script_dir}/install_and_cache_pkgs.sh ~/cache-apt-pkgs ${packages}
|
||||
fi
|
||||
echo ""
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ source "${script_dir}/lib.sh"
|
|||
cache_dir="${1}"
|
||||
|
||||
# Version of the cache to create or load.
|
||||
cache_version="${2}"
|
||||
version="${2}"
|
||||
|
||||
# List of the packages to use.
|
||||
input_packages="${@:3}"
|
||||
|
|
@ -17,57 +17,53 @@ input_packages="${@:3}"
|
|||
packages="$(normalize_package_list "${input_packages}")"
|
||||
|
||||
# Create cache directory so artifacts can be saved.
|
||||
mkdir -p "${cache_dir}"
|
||||
mkdir -p ${cache_dir}
|
||||
|
||||
echo -n "Validating action arguments (version='${cache_version}', packages='${packages}')...";
|
||||
if grep -q " " <<< "${cache_version}"; then
|
||||
echo "aborted."
|
||||
echo "Version value '${cache_version}' cannot contain spaces." >&2
|
||||
log -n "Validating action arguments (version='${version}', packages='${packages}')...";
|
||||
if grep -q " " <<< "${version}"; then
|
||||
log "aborted."
|
||||
log "Version value '${version}' cannot contain spaces." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Is length of string zero?
|
||||
if test -z "${packages}"; then
|
||||
echo "aborted."
|
||||
echo "Packages argument cannot be empty." >&2
|
||||
log "aborted."
|
||||
log "Packages argument cannot be empty." >&2
|
||||
exit 2
|
||||
fi
|
||||
echo "done."
|
||||
|
||||
echo -n "Updating APT package list..."
|
||||
sudo apt-get update > /dev/null
|
||||
echo "done."
|
||||
log "done."
|
||||
|
||||
versioned_packages=""
|
||||
echo -n "Verifying packages..."
|
||||
log -n "Verifying packages..."
|
||||
for package in ${packages}; do
|
||||
if test ! "$(apt show "${package}")"; then
|
||||
echo "aborted."
|
||||
echo "Package '${package}' not found." >&2
|
||||
log "Package '${package}' not found." >&2
|
||||
exit 3
|
||||
fi
|
||||
get_package_name_ver "${package}" # -> package_name, package_ver
|
||||
versioned_packages="${versioned_packages} ${package_name}=${package_ver}"
|
||||
read package_name package_ver < <(get_package_name_ver "${package}")
|
||||
versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}""
|
||||
done
|
||||
echo "done."
|
||||
|
||||
# Abort on any failure at this point.
|
||||
set -e
|
||||
|
||||
echo "Creating cache key..."
|
||||
log "Creating cache key..."
|
||||
|
||||
# TODO Can we prove this will happen again?
|
||||
normalized_versioned_packages="$(normalize_package_list "${versioned_packages}")"
|
||||
echo "- Normalized package list is '${normalized_versioned_packages}'."
|
||||
log "- Normalized package list is '${normalized_versioned_packages}'."
|
||||
|
||||
value="$(echo "${normalized_versioned_packages} @ ${cache_version}")"
|
||||
echo "- Value to hash is '${value}'."
|
||||
value="${normalized_versioned_packages} @ ${version}"
|
||||
log "- Value to hash is '${value}'."
|
||||
|
||||
key="$(echo "${value}" | md5sum | /bin/cut -f1 -d' ')"
|
||||
echo "- Value hashed as '${key}'."
|
||||
log "- Value hashed as '${key}'."
|
||||
|
||||
echo "done."
|
||||
log "done."
|
||||
|
||||
key_filepath="${cache_dir}/cache_key.md5"
|
||||
echo "${key}" > "${key_filepath}"
|
||||
echo "Hash value written to ${key_filepath}"
|
||||
echo ${key} > ${key_filepath}
|
||||
log "Hash value written to ${key_filepath}"
|
||||
|
|
|
|||
|
|
@ -3,32 +3,36 @@
|
|||
# Fail on any error.
|
||||
set -e
|
||||
|
||||
# Include library.
|
||||
script_dir="$(dirname -- "$(realpath -- "${0}")")"
|
||||
source "${script_dir}/lib.sh"
|
||||
|
||||
# Directory that holds the cached packages.
|
||||
cache_dir=$1
|
||||
cache_dir="${1}"
|
||||
|
||||
# Root directory to untar the cached packages to.
|
||||
# Typically filesystem root '/' but can be changed for testing.
|
||||
cache_restore_root=$2
|
||||
cache_restore_root="${2}"
|
||||
|
||||
cache_filepaths=$(ls -1 $cache_dir | sort)
|
||||
echo "Found $(echo $cache_filepaths | wc -w) files in the cache."
|
||||
for cache_filepath in $cache_filepaths; do
|
||||
echo "- $(basename $cache_filepath)"
|
||||
cache_filepaths="$(ls -1 "${cache_dir}" | sort)"
|
||||
log "Found $(echo ${cache_filepaths} | wc -w) files in the cache."
|
||||
for cache_filepath in ${cache_filepaths}; do
|
||||
log "- "$(basename ${cache_filepath})""
|
||||
done
|
||||
|
||||
echo "Reading from manifest..."
|
||||
for logline in "$(cat "${cache_dir}/manifest.log" | tr ',' '\n' )"; do
|
||||
echo "- $(echo "${logline}" | tr ':' ' ')"
|
||||
log "Reading from main requested packages manifest..."
|
||||
for logline in $(cat "${cache_dir}/manifest_main.log" | tr ',' '\n' ); do
|
||||
log "- $(echo "${logline}" | tr ':' ' ')"
|
||||
done
|
||||
echo "done."
|
||||
log "done."
|
||||
|
||||
# Only search for archived results. Manifest and cache key also live here.
|
||||
cache_pkg_filepaths=$(ls -1 $cache_dir/*.tar.gz | sort)
|
||||
cache_pkg_filecount=$(echo $cache_pkg_filepaths | wc -w)
|
||||
echo "Restoring $cache_pkg_filecount packages from cache..."
|
||||
for cache_pkg_filepath in $cache_pkg_filepaths; do
|
||||
echo -n "- $(basename $cache_pkg_filepath) restoring..."
|
||||
sudo tar -xf $cache_pkg_filepath -C $cache_restore_root > /dev/null
|
||||
echo "done."
|
||||
cache_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort)
|
||||
cache_pkg_filecount=$(echo ${cache_pkg_filepaths} | wc -w)
|
||||
log "Restoring ${cache_pkg_filecount} packages from cache..."
|
||||
for cache_pkg_filepath in ${cache_pkg_filepaths}; do
|
||||
log "- $(basename "${cache_pkg_filepath}") restoring..."
|
||||
sudo tar -xf "${cache_pkg_filepath}" -C "${cache_restore_root}" > /dev/null
|
||||
log "done."
|
||||
done
|
||||
echo "done."
|
||||
log "done."
|
||||
|
|
|
|||
Loading…
Reference in a new issue