Martin L Roth has uploaded this change for review. ( https://review.coreboot.org/c/coreboot/+/77580?usp=email )
Change subject: util/scripts: Improve update_submodules script ......................................................................
util/scripts: Improve update_submodules script
I recently tried to update the amd_blobs submodule with the script, and it tried to check out origin/main instead of origin/master which would have rolled the submodule pointer back one commit. the commit also added another submodule that was previously staged. These issues prompted me to fix this script so it wouldn't do unexpected things.
This adds a number of changes and fixes to the update_submodules script. - Update the version to 2.00 - Make global variables uppercase - Add pause function for warnings and errors - Add debug functions and -D | --debug argument to enable them - Add -b | --branch argument to allow branch to be specified - Add -m | --min_days argument to specify the min time (30 day default) - Split out functionality to find the branch name into a function - If there are multiple valid branches, try to pick the newest - Error out if no branches are valid - Make local variables local - Verify that nothing is staged that would get added accidentally - Verify that the submodule being checked is clean - Show when submodules are being fetched - Get 12 digits of commit ID for initial and ending commits - Print short date for initial & ending commits - Don't update if the last update was more recent than min_days - Make sure we're not updating to an older commit - If there's only one new commit, don't say 'commits' (plurl)
Signed-off-by: Martin Roth gaumless@gmail.com Change-Id: I39ae6799f2965e8421fb0ddfb463833b3f3be037 --- M util/scripts/update_submodules 1 file changed, 191 insertions(+), 52 deletions(-)
git pull ssh://review.coreboot.org:29418/coreboot refs/changes/80/77580/1
diff --git a/util/scripts/update_submodules b/util/scripts/update_submodules index 9f6c14f..1077b75 100755 --- a/util/scripts/update_submodules +++ b/util/scripts/update_submodules @@ -9,7 +9,7 @@
set -eu -o pipefail
-VERSION="1.01" +VERSION="2.00" PROGRAM=$0 PROGNAME="$(basename "${PROGRAM}")"
@@ -17,12 +17,29 @@ export LC_ALL=C export TZ=UTC0
-min_commits=10 +MIN_COMMITS=10 TOP=${PWD} SUBMODULES_WITH_UPDATES=0 -submodule_dirs=() -skip_sync="" -max_commits_to_list=65 +SUBMODULE_DIRS=() +SKIP_SYNC="" +MAX_COMMITS_TO_LIST=65 +INITIAL_BRANCH_NAME="" +BRANCH_NAME="" +DEBUG="" +MIN_SECONDS=2592000 # 30 days + +pause() { + if [ -n "$1" ]; then echo "$1"; fi + read -r -p " Press [Enter] key to continue..." +} + +debug() { + if [[ -n "${DEBUG}" ]]; then echo "$1"; fi +} + +debug_pause() { + if [[ -n "${DEBUG}" ]]; then pause "$1"; fi +}
show_version() { echo "${PROGNAME} version ${VERSION}" @@ -33,8 +50,11 @@ echo "Usage: ${PROGNAME} [options]" echo echo "Options:" + echo " -b | --branch <name> Specify the branch name" echo " -c | --changes <#> Specify the minimum number of changes to update a repo" + echo " -D | --debug Print debug information" echo " -h | --help Print usage and exit" + echo " -m | --min_days Specify the minimum number of days between commits" echo " -R | --repo <dir> Specify a single repo directory to update" echo " -s | --skipsync Assume that repos are already synced" echo " -V | --version Print the version and exit" @@ -42,7 +62,7 @@ }
get_args() { - args=$(getopt -l changes:,help,repo:,skipsync,version -o c:hR:sV -- "$@") + args=$(getopt -l branch:,changes:,debug,help,min_days:,repo:,skipsync,version -o b:c:Dhm:R:sV -- "$@") getopt_ret=$? eval set -- "${args}"
@@ -56,25 +76,39 @@ opt="$1" shift case "${opt}" in - -c | --changes) - min_commits="${1}" + -b | --branch) + INITIAL_BRANCH_NAME="${1}" shift ;; + -c | --changes) + MIN_COMMITS="${1}" + shift + ;; + -D | --debug) + if [[ -n "${DEBUG}" ]]; then + set -x + fi + DEBUG=1 + ;; -h | --help) usage exit 0 ;; - -R | --repo) - submodule_dirs=("$(readlink -f "${1}")") + -m | --min_days) + MIN_SECONDS="$(($1 * 86400))" shift - if [[ ! -d "${submodule_dirs[0]}" ]]; then - echo "Error: ${submodule_dirs[0]} is not valid." + ;; + -R | --repo) + SUBMODULE_DIRS=("$(readlink -f "${1}")") + shift + if [[ ! -d "${SUBMODULE_DIRS[0]}" ]]; then + echo "Error: ${SUBMODULE_DIRS[0]} is not valid." usage exit 1 fi ;; -s | --skipsync) - skip_sync=1 + SKIP_SYNC=1 ;; -V | --version) exit 0 @@ -86,70 +120,175 @@ done }
+get_branch() { + local submodule=$1 + local branch + local branchname_date + local branch_date + + declare -a branches=("origin/main" "origin/master" "origin/trunk") + for branch in "${branches[@]}"; do + # For some reason grep -q wasn't working the same as [[ -n git | grep ]]. + # grep -q wasn't finding origin/master or origin/main in the chromeec repo. + # This might be the pipefail issue that SC2143 mentions. + # shellcheck disable=SC2143 + if [[ -n "$(git branch -a | grep "${branch}")" ]]; then + debug "Found ${branch}" + if [[ -z "${BRANCH_NAME}" ]]; then + BRANCH_NAME="${branch}" + branchname_date="$(git log "${BRANCH_NAME}" -n 1 --format="%ct" --)" + else + echo "Warning: Both ${branch} and ${BRANCH_NAME} exist in ${submodule}." + branch_date="$(git log "${branch}" -n 1 --format="%ct" --)" + if [[ "${branchname_date}" -le "${branch_date}" ]]; then + BRANCH_NAME="${branch}" + branchname_date=${branch_date} + fi + echo " ${BRANCH_NAME} is selected because it is newer or the same as ${branch}." + echo " Press CTL-C to quit and specify the branch on the command line with -b | --branch." + pause " Continuing will use the ${BRANCH_NAME} branch." + fi + else + debug "${branch} does not exist in ${submodule}" + fi + done +} +
main() { + local submodule + show_version get_args "$@"
- if (( ${#submodule_dirs[@]} == 0 )); then - readarray -t submodule_dirs < <(git submodule foreach pwd | grep -v "Entering") + # Get list of all submodules if nothing is specified + if (( ${#SUBMODULE_DIRS[@]} == 0 )); then + readarray -t SUBMODULE_DIRS < <(git submodule foreach pwd | grep -v "Entering") fi
- for submodule in "${submodule_dirs[@]}"; do - echo "Checking submodule ${submodule}" - if ! cd "$submodule"; then - echo "Error: could not cd to $submodule" - exit 1 - fi + # Make sure nothing is staged that would get added to the commit accidentally + if [[ -n "$(git diff --name-only --cached)" ]]; then + echo "You have files staged. Exiting." + exit 1 + fi
- initial_commit_id="$(git log --pretty='%h' -n 1)" - initial_commit_description="$(git log --pretty='%ci - (%s)' -n 1)" - if [[ ${skip_sync} != "1" ]]; then - git fetch 2>/dev/null - fi + for submodule in "${SUBMODULE_DIRS[@]}"; do + local submodule_name="${submodule##*/}"
- declare -a branches=("origin/main" "origin/master" "origin/trunk") - for branch in "${branches[@]}"; do - if git branch -a | grep "${branch}" > /dev/null; then - branch_name="${branch}" - break - fi - done - - updated_commit_id="$(git log --pretty='%h' -n 1 "${branch_name}" -- )" - updated_commit_description="$(git log --pretty='%ci - (%s)' -n 1 "${updated_commit_id}")" - if [ "${initial_commit_id}" = "${updated_commit_id}" ]; then - echo "No updates for ${submodule}" + echo "Checking submodule ${submodule_name}" + BRANCH_NAME="${INITIAL_BRANCH_NAME}" + # verify that the submodule is clean, then change to its directory + if git status "${submodule%/*}" | grep -q "${submodule_name}" ; then + echo "${submodule} looks dirty." + pause "Please make sure it is on the current coreboot commit. Skipping." continue fi + + if ! cd "${submodule}"; then + pause "Error: Could not cd to ${submodule}" + continue + fi + + # Get the base commit ID & message + local initial_commit_id + local initial_commit_date + local initial_commit_description + initial_commit_id="$(git log --pretty='%h' --abbrev=12 -n 1 --)" + initial_commit_date="$(git log --pretty='%ct' -n 1 --)" + initial_commit_description="$(git log --pretty='(%cs):%n %s' -n 1 "${initial_commit_id}" --)" + + # Update the submodule's git repo and find the branch + if [[ ${SKIP_SYNC} != "1" ]]; then + echo " Fetching submodule ${submodule_name}" + git fetch 2>/dev/null + fi + if [[ -z "${BRANCH_NAME}" ]]; then + get_branch "${submodule_name}" + if [[ -z "${BRANCH_NAME}" ]]; then + echo "Error: Remote branch for ${submodule_name} not found." + echo " Specify the branch on the command line with -b | --branch." + pause " Moving to next submodule." + cd "${TOP}" || exit 1 + continue + fi + fi + + # Get the updated commit ID & message + local updated_commit_id + local updated_commit_date + local updated_commit_description + updated_commit_id="$(git log --pretty='%h' --abbrev=12 -n 1 "${BRANCH_NAME}" --)" + updated_commit_date="$(git log --pretty='%ct' -n 1 "${updated_commit_id}" --)" + updated_commit_description="$(git log --pretty='(%cs):%n %s' -n 1 "${updated_commit_id}" --)" + if [[ "${initial_commit_id}" = "${updated_commit_id}" ]]; then + printf " No updates for %s\n\n" "${submodule}" + cd "${TOP}" || exit 1 + continue + else + debug " ${initial_commit_id} != ${updated_commit_id}" + fi + + local now + now="$(date +%s)" + if [[ "${updated_commit_date}" -lt "${initial_commit_date}" ]]; then + echo "Error: Updated commit is earlier than the current commit." + pause " Skipping ${submodule_name}." + cd "${TOP}" || exit 1 + continue + elif [[ "$((now - initial_commit_date))" -lt "${MIN_SECONDS}" ]]; then + printf " Initial commit is too new to create an update.\n\n" + cd "${TOP}" || exit 1 + continue + else + debug " Commit Date: ${updated_commit_date} > ${initial_commit_date} ($((initial_commit_date - updated_commit_date)))" + fi + + # Create the new commit SUBMODULES_WITH_UPDATES+=1 - update_log="$(git log --oneline "${initial_commit_id}..${updated_commit_id}")" + local update_log + local update_count + update_log="$(git log --oneline --abbrev=10 "${initial_commit_id}..${updated_commit_id}")" update_count="$(echo "${update_log}" | wc -l)" - if [[ "${update_count}" -gt "${max_commits_to_list}" ]]; then + if [[ "${update_count}" -gt "${MAX_COMMITS_TO_LIST}" ]]; then update_log="" new_commit_terminator="." else new_commit_terminator=":" fi - echo "${update_count} new commits for ${submodule}" - if [ "${update_count}" -ge "${min_commits}" ]; then - echo "Creating commit to update ${submodule##*/} submodule" + + local plurl="" + if [[ "${update_count}" -ne 1 ]]; then + plurl="s" + fi + + echo " ${update_count} new commits for ${submodule}" + if [[ "${update_count}" -ge "${MIN_COMMITS}" ]]; then + echo " Creating commit to update ${submodule_name} submodule" git checkout "${updated_commit_id}" > /dev/null 2>&1 cd "${TOP}" || exit 1 git add "${submodule}" > /dev/null 2>&1 || exit 1 - git commit -s -F- > /dev/null 2>&1 <<-EOF - Update ${submodule##*/} submodule to upstream ${branch##*/}
- Updating from commit id ${initial_commit_id}: - $initial_commit_description + local commitinfo + local retval + commitinfo="$(git commit -s -F- 2>&1 <<-EOF + Update ${submodule_name} submodule to upstream ${BRANCH_NAME##*/} branch
- to commit id ${updated_commit_id}: - ${updated_commit_description} + Updating from commit id ${initial_commit_id} $initial_commit_description
- This brings in ${update_count} new commits${new_commit_terminator} - ${update_log} - EOF + to commit id ${updated_commit_id} ${updated_commit_description} + + This brings in ${update_count} new commit${plurl}${new_commit_terminator} + ${update_log} + + EOF + )"; retval=$? + if [[ "${retval}" -ne 0 ]]; then + echo "Error creating commit." + echo "${commitinfo}" + fi fi + echo + debug_pause "go to next submodule" done
if [ "${SUBMODULES_WITH_UPDATES}" = "0" ]; then