Patrick Georgi (patrick@georgi-clan.de) just uploaded a new patch set to gerrit, which you can find at http://review.coreboot.org/409
-gerrit
commit 89323d9f59044c9036f7ea96c7394606ee3f298e Author: Patrick Georgi patrick@georgi-clan.de Date: Sat Nov 5 14:44:41 2011 +0100
abuild: Build boards in parallel if possible
Determine if xargs -P works. If yes, use that to build multiple boards in parallel, instead of relying on make -j X, when doing a full abuild run (instead of single boards).
make -j X isn't able to make use of several cores at various serialization points in our build process, so this change results in a >25% speed up for a full abuild run in my tests.
Change-Id: Id484a4211c84a3a24115278e0fbe92345f346596 Signed-off-by: Patrick Georgi patrick@georgi-clan.de --- util/abuild/abuild | 68 +++++++++++++++++++++++++++++++++++++++------------ 1 files changed, 52 insertions(+), 16 deletions(-)
diff --git a/util/abuild/abuild b/util/abuild/abuild index 74d2908..83e84bf 100755 --- a/util/abuild/abuild +++ b/util/abuild/abuild @@ -563,6 +563,9 @@ test -f util/sconfig/sconfig.l && ROOT=$( pwd ) test -f ../util/sconfig/sconfig.l && ROOT=$( cd ..; pwd ) test "$ROOT" = "" && ROOT=$( cd ../..; pwd )
+# command line for xargs parallelization. Thus overwrite -c X +cmdline="$* -c 1" + # parse parameters.. try to find out whether we're running GNU getopt getoptbrand="`getopt -V`" if [ "${getoptbrand:0:6}" == "getopt" ]; then @@ -607,8 +610,43 @@ while true ; do esac done
+USE_XARGS=0 if [ "$cpus" != "1" ]; then - export MAKEFLAGS="-j $cpus" + if [ "$target" = "" ]; then + # Test if xargs supports the non-standard -P flag + echo | xargs -P 0$cpus -n 1 echo 2>/dev/null >/dev/null && USE_XARGS=1 + fi +fi + +if [ "$USE_XARGS" = "0" ]; then +export MAKEFLAGS="-j $cpus" +build_all_targets() +{ + for VENDOR in $( vendors ); do + for MAINBOARD in $( mainboards $VENDOR ); do + build_target $VENDOR $MAINBOARD + test_target $VENDOR $MAINBOARD + remove_target $VENDOR $MAINBOARD + done + done +} +else +# Limit to 32 parallel builds for now. +# Thrashing all caches because we run +# 160 abuilds in parallel is no fun. +if [ "$cpus" = "" ]; then + cpus=32 +fi +build_all_targets() +{ + # seed shared utils + MAKEFLAGS="-j $cpus" $0 $cmdline -t emulation/qemu-x86 + for VENDOR in $( vendors ); do + for MAINBOARD in $( mainboards $VENDOR ); do + echo $VENDOR/$MAINBOARD + done + done | xargs -P 0$cpus -n 1 $0 $cmdline -t +} fi
# /path/to/freebios2/ @@ -629,23 +667,21 @@ if [ "$target" != "" ]; then CONFIG=`printf $target|cut -f3 -d/` if [ ! -r $ROOT/src/mainboard/$target ]; then printf "No such target: $target\n" - xml '</abuild>' - junit '</testsuite>' - exit 1 + failed=1 + else + build_target $VENDOR $MAINBOARD $CONFIG + test_target $VENDOR $MAINBOARD + test "$mode" != "text" && cat $TARGET/${VENDOR}_${MAINBOARD}/abuild.xml >> $REAL_XMLFILE + XMLFILE=$REAL_XMLFILE fi - build_target $VENDOR $MAINBOARD $CONFIG - test_target $VENDOR $MAINBOARD - test "$mode" != "text" && cat $TARGET/${VENDOR}_${MAINBOARD}/abuild.xml >> $REAL_XMLFILE - XMLFILE=$REAL_XMLFILE else - # build all boards per default - for VENDOR in $( vendors ); do - for MAINBOARD in $( mainboards $VENDOR ); do - build_target $VENDOR $MAINBOARD - test_target $VENDOR $MAINBOARD - remove_target $VENDOR $MAINBOARD - done - done + build_all_targets + rm -f $REAL_XMLFILE + xml '<?xml version="1.0" encoding="utf-8"?>' + xml '<abuild>' + + junit '<?xml version="1.0" encoding="utf-8"?>' + junit '<testsuite>' if [ "$mode" != "text" ]; then for xmlfile in $TARGET/*_*/abuild.xml; do cat $xmlfile >> $REAL_XMLFILE