Patrick Georgi (patrick(a)georgi-clan.de) just uploaded a new patch set to gerrit, which you can find at http://review.coreboot.org/409
-gerrit
commit ff379c8f24cebaf789311838e3c9fcdfd7ef198d
Author: Patrick Georgi <patrick(a)georgi-clan.de>
Date: Sat Nov 5 14:44:41 2011 +0100
abuild: Build boards in parallel if possible
Determine if xargs -P works. If yes, use that to build multiple
boards in parallel, instead of relying on make -j X, when doing
a full abuild run (instead of single boards).
make -j X isn't able to make use of several cores at various
serialization points in our build process, so this change results
in a >25% speed up for a full abuild run in my tests.
Change-Id: Id484a4211c84a3a24115278e0fbe92345f346596
Signed-off-by: Patrick Georgi <patrick(a)georgi-clan.de>
---
util/abuild/abuild | 68 +++++++++++++++++++++++++++++++++++++++------------
1 files changed, 52 insertions(+), 16 deletions(-)
diff --git a/util/abuild/abuild b/util/abuild/abuild
index 8b81ea1..6acd793 100755
--- a/util/abuild/abuild
+++ b/util/abuild/abuild
@@ -583,6 +583,9 @@ test -f util/sconfig/sconfig.l && ROOT=$( pwd )
test -f ../util/sconfig/sconfig.l && ROOT=$( cd ..; pwd )
test "$ROOT" = "" && ROOT=$( cd ../..; pwd )
+# command line for xargs parallelization. Thus overwrite -c X
+cmdline="$* -c 1"
+
# parse parameters.. try to find out whether we're running GNU getopt
getoptbrand="`getopt -V`"
if [ "${getoptbrand:0:6}" == "getopt" ]; then
@@ -628,8 +631,43 @@ while true ; do
esac
done
+USE_XARGS=0
if [ "$cpus" != "1" ]; then
- export MAKEFLAGS="-j $cpus"
+ if [ "$target" = "" ]; then
+ # Test if xargs supports the non-standard -P flag
+ echo | xargs -P 0$cpus -n 1 echo 2>/dev/null >/dev/null && USE_XARGS=1
+ fi
+fi
+
+if [ "$USE_XARGS" = "0" ]; then
+export MAKEFLAGS="-j $cpus"
+build_all_targets()
+{
+ for VENDOR in $( vendors ); do
+ for MAINBOARD in $( mainboards $VENDOR ); do
+ build_target $VENDOR $MAINBOARD
+ test_target $VENDOR $MAINBOARD
+ remove_target $VENDOR $MAINBOARD
+ done
+ done
+}
+else
+# Limit to 32 parallel builds for now.
+# Thrashing all caches because we run
+# 160 abuilds in parallel is no fun.
+if [ "$cpus" = "" ]; then
+ cpus=32
+fi
+build_all_targets()
+{
+ # seed shared utils
+ MAKEFLAGS="-j $cpus" $0 $cmdline -t emulation/qemu-x86
+ for VENDOR in $( vendors ); do
+ for MAINBOARD in $( mainboards $VENDOR ); do
+ echo $VENDOR/$MAINBOARD
+ done
+ done | xargs -P 0$cpus -n 1 $0 $cmdline -t
+}
fi
# /path/to/freebios2/
@@ -650,23 +688,21 @@ if [ "$target" != "" ]; then
CONFIG=`printf $target|cut -f3 -d/`
if [ ! -r $ROOT/src/mainboard/$target ]; then
printf "No such target: $target\n"
- xml '</abuild>'
- junit '</testsuite>'
- exit 1
+ failed=1
+ else
+ build_target $VENDOR $MAINBOARD $CONFIG
+ test_target $VENDOR $MAINBOARD
+ cat $TARGET/${VENDOR}_${MAINBOARD}/abuild.xml >> $REAL_XMLFILE
+ XMLFILE=$REAL_XMLFILE
fi
- build_target $VENDOR $MAINBOARD $CONFIG
- test_target $VENDOR $MAINBOARD
- cat $TARGET/${VENDOR}_${MAINBOARD}/abuild.xml >> $REAL_XMLFILE
- XMLFILE=$REAL_XMLFILE
else
- # build all boards per default
- for VENDOR in $( vendors ); do
- for MAINBOARD in $( mainboards $VENDOR ); do
- build_target $VENDOR $MAINBOARD
- test_target $VENDOR $MAINBOARD
- remove_target $VENDOR $MAINBOARD
- done
- done
+ build_all_targets
+ rm -f $REAL_XMLFILE
+ xml '<?xml version="1.0" encoding="utf-8"?>'
+ xml '<abuild>'
+
+ junit '<?xml version="1.0" encoding="utf-8"?>'
+ junit '<testsuite>'
for xmlfile in $TARGET/*_*/abuild.xml; do
cat $xmlfile >> $REAL_XMLFILE
done
Patrick Georgi (patrick(a)georgi-clan.de) just uploaded a new patch set to gerrit, which you can find at http://review.coreboot.org/403
-gerrit
commit 4162c3bbf3d0be1cf2cf71ab408d3b65de75a0bc
Author: Patrick Georgi <patrick(a)georgi-clan.de>
Date: Sat Nov 5 01:18:53 2011 +0100
Allow using binary libpayload
We already test for LIBPAYLOAD_PREFIX in the Makefile's build codepath.
Also test for it in the configuration codepath, so a prebuilt libpayload
can be used for FILO.
Change-Id: I8f2aa102f1e41f698bfd4061c5e7e7b8082c885e
Signed-off-by: Patrick Georgi <patrick(a)georgi-clan.de>
---
util/kconfig/Makefile | 6 ++++++
1 files changed, 6 insertions(+), 0 deletions(-)
diff --git a/util/kconfig/Makefile b/util/kconfig/Makefile
index 70c3870..1d519b2 100644
--- a/util/kconfig/Makefile
+++ b/util/kconfig/Makefile
@@ -18,10 +18,16 @@ LIB_CONFIG := $(src)/lib.config
HAVE_FILO_CONFIG := $(wildcard $(FILO_CONFIG))
HAVE_LIB_CONFIG := $(wildcard $(LIB_CONFIG))
+LIBPAYLOAD ?= $(wildcard $(LIBPAYLOAD_PREFIX)/lib/libpayload.a)
+ifeq ($(LIBPAYLOAD),)
libpayload_config_%:
echo "Libpayload config for FILO."
$(MAKE) -C $(LIBCONFIG_PATH) KCONFIG_CONFIG=$(LIB_CONFIG) $(patsubst libpayload_config_%,%,$@)
echo "Libpayload config done."
+else
+libpayload_config_%:
+ echo "Using binary libpayload, nothing to configure"
+endif
xconfig: prepare $(objk)/qconf libpayload_config_xconfig
$(objk)/qconf $(Kconfig)
Patrick Georgi (patrick(a)georgi-clan.de) just uploaded a new patch set to gerrit, which you can find at http://review.coreboot.org/409
-gerrit
commit 991d0403fc05ab37d61078d98a6edd9245137c54
Author: Patrick Georgi <patrick(a)georgi-clan.de>
Date: Sat Nov 5 14:44:41 2011 +0100
abuild: Build boards in parallel if possible
Determine if xargs -P works. If yes, use that to build multiple
boards in parallel, instead of relying on make -j X, when doing
a full abuild run (instead of single boards).
make -j X isn't able to make use of several cores at various
serialization points in our build process, so this change results
in a >25% speed up for a full abuild run in my tests.
Change-Id: Id484a4211c84a3a24115278e0fbe92345f346596
Signed-off-by: Patrick Georgi <patrick(a)georgi-clan.de>
---
util/abuild/abuild | 49 ++++++++++++++++++++++++++++++++++++++++---------
1 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/util/abuild/abuild b/util/abuild/abuild
index 8b81ea1..5228cdb 100755
--- a/util/abuild/abuild
+++ b/util/abuild/abuild
@@ -583,6 +583,9 @@ test -f util/sconfig/sconfig.l && ROOT=$( pwd )
test -f ../util/sconfig/sconfig.l && ROOT=$( cd ..; pwd )
test "$ROOT" = "" && ROOT=$( cd ../..; pwd )
+# command line for xargs parallelization. Thus overwrite -c X
+cmdline="$* -c 1"
+
# parse parameters.. try to find out whether we're running GNU getopt
getoptbrand="`getopt -V`"
if [ "${getoptbrand:0:6}" == "getopt" ]; then
@@ -628,8 +631,43 @@ while true ; do
esac
done
+USE_XARGS=0
if [ "$cpus" != "1" ]; then
- export MAKEFLAGS="-j $cpus"
+ if [ "$target" = "" ]; then
+ # Test if xargs supports the non-standard -P flag
+ echo | xargs -P 0$cpus -n 1 echo 2>/dev/null >/dev/null && USE_XARGS=1
+ fi
+fi
+
+if [ "$USE_XARGS" = "0" ]; then
+export MAKEFLAGS="-j $cpus"
+build_all_targets()
+{
+ for VENDOR in $( vendors ); do
+ for MAINBOARD in $( mainboards $VENDOR ); do
+ build_target $VENDOR $MAINBOARD
+ test_target $VENDOR $MAINBOARD
+ remove_target $VENDOR $MAINBOARD
+ done
+ done
+}
+else
+# Limit to 32 parallel builds for now.
+# Thrashing all caches because we run
+# 160 abuilds in parallel is no fun.
+if [ "$cpus" = "" ]; then
+ cpus=32
+fi
+build_all_targets()
+{
+ # seed shared utils
+ MAKEFLAGS="-j $cpus" $0 $cmdline -t emulation/qemu-x86
+ for VENDOR in $( vendors ); do
+ for MAINBOARD in $( mainboards $VENDOR ); do
+ echo $VENDOR/$MAINBOARD
+ done
+ done | xargs -P 0$cpus -n 1 $0 $cmdline -t
+}
fi
# /path/to/freebios2/
@@ -659,14 +697,7 @@ if [ "$target" != "" ]; then
cat $TARGET/${VENDOR}_${MAINBOARD}/abuild.xml >> $REAL_XMLFILE
XMLFILE=$REAL_XMLFILE
else
- # build all boards per default
- for VENDOR in $( vendors ); do
- for MAINBOARD in $( mainboards $VENDOR ); do
- build_target $VENDOR $MAINBOARD
- test_target $VENDOR $MAINBOARD
- remove_target $VENDOR $MAINBOARD
- done
- done
+ build_all_targets
for xmlfile in $TARGET/*_*/abuild.xml; do
cat $xmlfile >> $REAL_XMLFILE
done
Patrick Georgi (patrick(a)georgi-clan.de) just uploaded a new patch set to gerrit, which you can find at http://review.coreboot.org/408
-gerrit
commit 82585042cc979c553aee4d9e88665b4b401d342f
Author: Patrick Georgi <patrick(a)georgi-clan.de>
Date: Sat Nov 5 14:39:56 2011 +0100
kconfig: Use more collision resistant temporary filenames
kconfig creates reasonably safe filenames for its temporary files
except for two of them.
Change-Id: I6861f55ae2a5311e3fb7919333ce9af1e39ce78b
Signed-off-by: Patrick Georgi <patrick(a)georgi-clan.de>
---
util/kconfig/confdata.c | 11 +++++++----
1 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/util/kconfig/confdata.c b/util/kconfig/confdata.c
index 946b8a9..79e0b35 100644
--- a/util/kconfig/confdata.c
+++ b/util/kconfig/confdata.c
@@ -682,6 +682,7 @@ int conf_write_autoconf(void)
FILE *out, *out_h;
time_t now;
int i, l;
+ char tmpname[128], tmpname_h[128];
sym_clear_all_valid();
@@ -692,11 +693,13 @@ int conf_write_autoconf(void)
return 1;
#endif
- out = fopen(".tmpconfig", "w");
+ sprintf(tmpname, ".tmpconfig.%d", (int)getpid());
+ out = fopen(tmpname, "w");
if (!out)
return 1;
- out_h = fopen(".tmpconfig.h", "w");
+ sprintf(tmpname_h, ".tmpconfig.h.%d", (int)getpid());
+ out_h = fopen(tmpname_h, "w");
if (!out_h) {
fclose(out);
return 1;
@@ -791,7 +794,7 @@ int conf_write_autoconf(void)
if (!name)
name = "include/linux/autoconf.h";
UNLINK_IF_NECESSARY(name);
- if (rename(".tmpconfig.h", name))
+ if (rename(tmpname_h, name))
return 1;
name = getenv("KCONFIG_AUTOCONFIG");
if (!name)
@@ -801,7 +804,7 @@ int conf_write_autoconf(void)
* and this marks the successful completion of the previous steps.
*/
UNLINK_IF_NECESSARY(name);
- if (rename(".tmpconfig", name))
+ if (rename(tmpname, name))
return 1;
return 0;
Patrick Georgi (patrick(a)georgi-clan.de) just uploaded a new patch set to gerrit, which you can find at http://review.coreboot.org/407
-gerrit
commit 6e67db0f0212f219c0eca3cae45ceff0a1abd79a
Author: Patrick Georgi <patrick(a)georgi-clan.de>
Date: Sat Nov 5 13:21:14 2011 +0100
abuild: Write XML/JUnit files per board
Write them per-board and merge them after everything is done.
This prepares for build parallelization.
Change-Id: Ia4e7ce03473bcf8861fb9ae06e9c1270292401ac
Signed-off-by: Patrick Georgi <patrick(a)georgi-clan.de>
---
util/abuild/abuild | 12 +++++++++++-
1 files changed, 11 insertions(+), 1 deletions(-)
diff --git a/util/abuild/abuild b/util/abuild/abuild
index 4ab0230..8b81ea1 100755
--- a/util/abuild/abuild
+++ b/util/abuild/abuild
@@ -17,9 +17,12 @@
ABUILD_DATE="December 10th, 2010"
ABUILD_VERSION="0.9.1"
+TOP=$PWD
+
# Where shall we place all the build trees?
TARGET=coreboot-builds
-XMLFILE=$( pwd )/abuild.xml
+XMLFILE=$TOP/abuild.xml
+REAL_XMLFILE=$XMLFILE
# path to payload. Should be more generic
PAYLOAD=/dev/null
@@ -363,6 +366,7 @@ function build_target
HOSTCC='gcc'
printf "Building $VENDOR/$MAINBOARD; "
+ XMLFILE=$TOP/$TARGET/${VENDOR}_${MAINBOARD}/abuild.xml
xml "<mainboard>"
xml ""
@@ -652,6 +656,8 @@ if [ "$target" != "" ]; then
fi
build_target $VENDOR $MAINBOARD $CONFIG
test_target $VENDOR $MAINBOARD
+ cat $TARGET/${VENDOR}_${MAINBOARD}/abuild.xml >> $REAL_XMLFILE
+ XMLFILE=$REAL_XMLFILE
else
# build all boards per default
for VENDOR in $( vendors ); do
@@ -661,6 +667,10 @@ else
remove_target $VENDOR $MAINBOARD
done
done
+ for xmlfile in $TARGET/*_*/abuild.xml; do
+ cat $xmlfile >> $REAL_XMLFILE
+ done
+ XMLFILE=$REAL_XMLFILE
fi
xml '</abuild>'
junit '</testsuite>'
Patrick Georgi (patrick(a)georgi-clan.de) just uploaded a new patch set to gerrit, which you can find at http://review.coreboot.org/405
-gerrit
commit 4e95b5a8d0a0c82705ff8cadf1f316278dc02dd1
Author: Patrick Georgi <patrick(a)georgi-clan.de>
Date: Sat Nov 5 12:47:13 2011 +0100
abuild: Avoid race condition when running abuild parallely
By moving the just-created file away, parallel runs of abuild might break.
Change-Id: I03368f00e9b11dad4c80d41279970e28debc7ed5
Signed-off-by: Patrick Georgi <patrick(a)georgi-clan.de>
---
util/abuild/abuild | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/util/abuild/abuild b/util/abuild/abuild
index 0ed295f..ba144aa 100755
--- a/util/abuild/abuild
+++ b/util/abuild/abuild
@@ -289,7 +289,7 @@ function compile_target
eval $MAKE $silent -j $cpus DOTCONFIG=${build_dir}/config.build obj=${build_dir} objutil=$TARGET/sharedutils \
&> ${build_dir}/make.log
ret=$?
- mv .xcompile ${build_dir}/xcompile.build
+ cp .xcompile ${build_dir}/xcompile.build
cd $TARGET/${VENDOR}_${MAINBOARD}
etime=`perl -e 'print time();' 2>/dev/null || date +%s`