Martin Roth has uploaded this change for review. ( https://review.coreboot.org/c/coreboot/+/59169 )
Change subject: utils: Add initial version of "remove_unused_code" script ......................................................................
utils: Add initial version of "remove_unused_code" script
This script creates a patch to remove all of the coreboot code that a platform doesn't use. This is useful for auditing the codebase for an individual platform or releasing a platform's code.
Unlike the script that Sage used that did something similar, this keeps the entire Kconfig tree (Though in a single file), all makefiles that are required to build, and the standard build tools can still be used. This will allow for much easier re-integration back into the coreboot codebase if code is released after running this.
This is just the initial version and more features needed to be added to make it fully functional. - It should be able to build multiple configurations to retain the code for all of those configurations. - Flag to remove submodules files as well - Additional variable flags to replace hardcoded values. - The list of makefiles that need to be kept is pretty long, and could be updated so that they aren't needed by the top level makefiles. - Add flag to show changed files - Show number of files before and after script is run
Signed-off-by: Martin Roth gaumless@gmail.com Change-Id: Iec69db2ad1358846d649db627b6d60ac8c2204e4 --- M util/scripts/description.md A util/scripts/rm_unused_code 2 files changed, 295 insertions(+), 0 deletions(-)
git pull ssh://review.coreboot.org:29418/coreboot refs/changes/69/59169/1
diff --git a/util/scripts/description.md b/util/scripts/description.md index a08771d..d028323 100644 --- a/util/scripts/description.md +++ b/util/scripts/description.md @@ -23,5 +23,7 @@ headers `Shell` * _parse-maintainers.pl_ - Script to alphabetize MAINTAINERS file `Perl` + * _rm_unused_code_ - Remove all code not used for a platform from the local + git repository for auditing or release `Bash` * _ucode_h_to_bin.sh_ - Microcode conversion tool `Bash` * _update_submodules_ - Check all submodules for updates `Bash` diff --git a/util/scripts/rm_unused_code b/util/scripts/rm_unused_code new file mode 100755 index 0000000..7c06959 --- /dev/null +++ b/util/scripts/rm_unused_code @@ -0,0 +1,293 @@ +#!/bin/bash +set -e + +VERSION="1.00" + +PROGRAM=$0 +PROGNAME="$(basename "${PROGRAM}")" + +MODIFIED_FILES= +CLEAN_DIR_LIST="configs Documentation payloads spd src util" +KEEP_FILES='util/kconfig/' +REQUIRED_MAKEFILES="util/testing|util/crossgcc|payloads/coreinfo|payloads/nvramcui|payloads/libpayload|payloads/external/tint|util/amdfwtool|util/ectool|util/futility|util/intelmetool|util/inteltool|util/intelvbttool|til/post|util/superiotool" +VERBOSE= +# Text STYLE variables +BOLD="\033[1m" +RED='\033[38;5;9m' +GREEN='\033[38;5;2m' +ORANGERED="\033[38;5;202m" +NO_COLOR='\033[0m' + +################################################################################ + +usage() { + echo "Usage: ${PROGNAME} [options]" + echo + echo "Options:" + echo ' -b | --blddir <dir> Set /tmp/<dir> as the build directory' + echo " -h | --help Print usage and exit" + echo " -D | --debug Print debug information. Use -DD to show all commands" + echo " -V | --version Print the version and exit" + echo " --nocolor Don't print color codes" + echo +} + +show_version() { + echo + _echo_color "${BOLD}${GREEN}" "${PROGNAME} version ${VERSION}" 1 + echo +} + +_echo_color() { + if [ "${3}" = "0" ]; then + printf "${1}%s${NO_COLOR}" "${2}" + else + printf "${1}%s${NO_COLOR}\n" "${2}" + fi +} + +_echo_debug() { + test -n "${VERBOSE}" && + printf "${ORANGERED}%s${NO_COLOR}\n" "${2}" >&2 +} + +_echo_error() { + (_echo_color >&2 "${RED}" "$*" 1) +} + +get_args() { + args=$(getopt -l version,help,debug,nocolor,blddir: -o b:DhV -- "$@") + getopt_ret=$? + eval set -- "${args}" + + if [ ${getopt_ret} != 0 ]; then + usage + exit 1 + fi + + while true; do + case "$1" in + -b | --blddir) + shift + BLD_DIR="/tmp/$1" + ;; + -D | --debug) + shift + # -d prints extra debug info + # -dd prints all script steps + if [ -n "${VERBOSE}" ]; then + set -x + else + VERBOSE="V=1" + fi + ;; + -h | --help) + shift + usage + exit 0 + ;; + --nocolor) + shift + BOLD="" + RED="" + GREEN="" + ORANGERED="" + NO_COLOR="" + ;; + -V | --version) exit 0 ;; + --) + shift + break + ;; + *) break ;; + esac + done + + BLD_DIR="${BLD_DIR:-$(mktemp -d)}" +} + +recursively_rm_dir_onlyfile() { + local dir=$1 + local beforecount + local aftercount + + while true; do + if [[ ! -d ${dir} ]]; then + break + fi + beforecount="$(find "${dir}" | wc -l)" + # shellcheck disable=SC2016 + find "${dir}" -depth -type d -exec /bin/bash -c \ + 'if echo "$1" | grep -q "$2"; then exit; fi && + if [[ "$(cd "$1" && find . -maxdepth 1 | grep -v "./Makefile")" = "." ]]; then + rm -rf "$1"; fi' shell {} \ + ${REQUIRED_MAKEFILES} ; + if [[ ! -d ${dir} ]]; then + break + fi + find "${dir}" -type d -empty -delete + if [[ ! -d ${dir} ]]; then + break + fi + aftercount="$(find "${dir}" | wc -l)" + if [[ ${aftercount} -eq ${beforecount} ]]; then + break + fi + done +} + +verify_atime_enabled() { + local testfile + # Make sure the build directory is mounted correctly + if [ ! -d "${BLD_DIR}" ]; then + mkdir "${BLD_DIR}" + fi + if ! grep -q "${BLD_DIR}" /proc/mounts; then + echo "Mounting the ${BLD_DIR} directory with atime enabled" + sudo mount -t tmpfs -o rw,relatime tmpfs "${BLD_DIR}" + elif ! grep "${BLD_DIR}" /proc/mounts | grep -q relatime; then + echo "Remounting the ${BLD_DIR} directory with relatime enabled" + sudo mount -o remount,relatime "${BLD_DIR}" + fi + + testfile="$(mktemp -p "${BLD_DIR}")" + touch -a --date="2020-01-01 00:00:00" "${testfile}" + if ! stat "${testfile}" | grep -q "Access: 2020-01-01"; then + _echo_error "Error: could not set access time." + sudo umount "${BLD_DIR}" + rm -rf "${BLD_DIR}" + exit 1 + fi + rm -f "${testfile}" +} + +update_codebase() { + local tempconfig + tempconfig="$(mktemp)" + if [ ! -f "${BLD_DIR}/COPYING" ]; then + echo "Downloading coreboot tree" + git clone https://review.coreboot.org/coreboot.git "${BLD_DIR}" + make -C "${BLD_DIR}" build/xcompile + fi + + # Start from a completely clean tree or we'll miss anything that + # doesn't need to be rebuilt. Save the config if it exists. + if [[ -f .config ]]; then + mv .config "${tempconfig}" + fi + _echo_color "${GREEN}" "Cleaning coreboot tree" + make -s -C "${BLD_DIR}" distclean + if [[ -f ${tempconfig} ]]; then + mv "${tempconfig}" .config + fi + + # force a refresh of all submodules + _echo_color "${GREEN}" "Refreshing all submodules..." + git submodule update --recursive --remote --init --checkout +} + +save_kconfig() { + (cd "${BLD_DIR}" && util/lint/kconfig_lint -w -p -o kconfig.tmp) +} + +update_times() { + _echo_color "${GREEN}" "Updating access time of all files" + git ls-files | xargs touch -a -m -t 202001010000 + # find "${BLD_DIR}" -type f -exec touch -a --date="2020-01-01 00:00:00" {} ; + if ! stat "${BLD_DIR}/COPYING" | grep -q "Access: 2020-01-01"; then + _echo_error "Error: could not set access time." + _echo_error " One of the following processes may be accessing it." + fuser -uvm "${BLD_DIR}/COPYING" + exit 1 + fi +} +mark_files_to_keep() { + for file in ${KEEP_FILES}; do + find "${BLD_DIR}/${file}" -depth -exec touch {} ; + done +} + +build_platform() { + local extra_text=$1 + _echo_color "${GREEN}" "Building platform ${extra_text}" + if [[ ! -f "${BLD_DIR}/.config" ]]; then + if [[ -n ${CONFIG_FILE} ]]; then + cp "${CONFIG_FILE}" "${BLD_DIR}/.config" || exit 1 + fi + echo "CONFIG_PAYLOAD_NONE=y" >>"${BLD_DIR}/.config" + fi + + make -C "${BLD_DIR}" -s clean UPDATED_SUBMODULES=1 BUILD_TIMELESS=1 + make -C "${BLD_DIR}" -s olddefconfig || exit 1 + make -C "${BLD_DIR}" -s UPDATED_SUBMODULES=1 BUILD_TIMELESS=1 ${VERBOSE} + HASH="$(sha256sum build/coreboot.rom)" + make -C "${BLD_DIR}" -s clean UPDATED_SUBMODULES=1 BUILD_TIMELESS=1 +} + +show_modified() { + MODIFIED_FILES=$(find "${BLD_DIR}" -atime -1 -type f | grep -v '.git') + echo "Files changed: $(echo "${MODIFIED_FILES}" | wc -l)" +} + +remove_kconfigs() { + # Dump all Kconfigs into a single file so that directories + # can be removed, while maintaining the entire Kconfig + # structure. + find "${BLD_DIR}/src" -name 'Kconfig*' -exec rm {} ; + mv "${BLD_DIR}/kconfig.tmp" "${BLD_DIR}/src/Kconfig" +} + +remove_unused() { + local dir + # Most files can be removed simply by looking at the time, but + # all Kconfig and Makefile.inc files in the entire tree are accessed + # whether they're used or not. + remove_kconfigs + + echo + _echo_color "${GREEN}" "Checking access time and removing unused files in:" + for dir in ${CLEAN_DIR_LIST}; do + printf "%s\n" "${BLD_DIR}/${dir}" + # find and remove all files without updated times. + find "${BLD_DIR}/${dir}" -atime +5 -type f -exec rm {} ; + + recursively_rm_dir_onlyfile "${BLD_DIR}/${dir}" + done + printf "\n\n" +} + +create_patch() { + _echo_color "${GREEN}" "Creating patch" + ( + cd "${BLD_DIR}" || exit 1 + git add -A + git commit -m "remove unused files" --no-verify && + git format-patch HEAD^ + ) +} + +main() { + show_version + get_args "$@" + + verify_atime_enabled + update_codebase + save_kconfig + update_times + mark_files_to_keep + build_platform "to mark used files" + OLDHASH=${HASH} + HASH="" + #show_modified + remove_unused + create_patch + build_platform "to verify the build still works" + NEWHASH=${HASH} + + echo + _echo_color "${GREEN}" "Checksums:" + echo "Old: ${OLDHASH}" + echo "New: ${NEWHASH}" +} + +main "$@"
-
Martin Roth (Code Review)