Nico Huber has uploaded this change for review. ( https://review.coreboot.org/23005
Change subject: [WIP] Add manibuilder ......................................................................
[WIP] Add manibuilder
Add a set of Dockerfiles for build testing. If you have an x86 machine and ~20GiB free disk space, run `make register` and `make -jxx` in util/manibuilder and go eat some pizza. The former runs a privileged docker container to set binfmt_misc up for qemu (read the code, don't trust it).
Change-Id: I60863a5c7d70dde71486fccb66cb59b30ba4d982 Signed-off-by: Nico Huber nico.h@gmx.de --- A util/manibuilder/Dockerfile.alpine A util/manibuilder/Dockerfile.anita A util/manibuilder/Dockerfile.centos A util/manibuilder/Dockerfile.debian-debootstrap A util/manibuilder/Dockerfile.fedora A util/manibuilder/Dockerfile.qemu-user-static A util/manibuilder/Dockerfile.ubuntu-debootstrap A util/manibuilder/Makefile A util/manibuilder/Makefile.targets A util/manibuilder/anita-wrapper.sh A util/manibuilder/mani-wrapper.sh 11 files changed, 341 insertions(+), 0 deletions(-)
git pull ssh://review.coreboot.org:29418/flashrom refs/changes/05/23005/1
diff --git a/util/manibuilder/Dockerfile.alpine b/util/manibuilder/Dockerfile.alpine new file mode 100644 index 0000000..63d4ba3 --- /dev/null +++ b/util/manibuilder/Dockerfile.alpine @@ -0,0 +1,17 @@ +FROM manibase + +RUN \ + adduser -D mani mani && \ + apk update && \ + apk add build-base linux-headers git ccache \ + pciutils-dev libusb-compat-dev libusb-dev + +USER mani +RUN \ + cd && \ + mkdir .ccache && chown mani:mani .ccache && \ + git clone https://review.coreboot.org/flashrom.git + +ENV DEVSHELL /bin/sh +COPY mani-wrapper.sh /home/mani/ +ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"] diff --git a/util/manibuilder/Dockerfile.anita b/util/manibuilder/Dockerfile.anita new file mode 100644 index 0000000..71f830d --- /dev/null +++ b/util/manibuilder/Dockerfile.anita @@ -0,0 +1,41 @@ +FROM debian:stable + +ARG PKG_PATH=http://cdn.netbsd.org/pub/pkgsrc/packages/NetBSD/amd64/7.1/All +ARG INST_IMG=http://ftp.de.netbsd.org/pub/NetBSD/NetBSD-7.1/amd64/ +ARG EXTRA_PKG="ccache libftdi" + +RUN \ + useradd -p locked -m mani && \ + apt-get -qq update && \ + apt-get -qq upgrade && \ + apt-get -qq dist-upgrade && \ + apt-get -qqy install git python python-pexpect \ + genisoimage qemu-system && \ + apt-get clean && \ + git clone https://github.com/gson1703/anita.git && \ + cd anita && python setup.py install + +USER mani +RUN cd && mkdir .ccache && chown mani:mani .ccache && \ + anita --sets kern-GENERIC,modules,base,etc,comp \ + --disk-size 1024M --memory-size=128M install ${INST_IMG} && \ + rm -rf work-*/download && \ + dd if=/dev/zero bs=1M count=64 of=cache.img && \ + anita --vmm-args '-hdb cache.img' --persist --run \ +"echo fdisk -0 -s 169/63/131009 -f -u wd1 && \ + newfs wd1a && \ + mkdir .ccache && \ + echo 'mount /dev/wd1a .ccache' >init && \ + echo dhcpcd >>init && \ + echo export PKG_PATH=${PKG_PATH} >>init && . ./init && \ + pkg_add gmake git-base pciutils libusb libusb1 ${EXTRA_PKG} && \ + git config --global --add http.sslVerify false && \ + git clone https://review.coreboot.org/flashrom.git && \ + ccache -M 60M; \ + umount .ccache" \ + boot ${INST_IMG} && \ + gzip cache.img + +COPY anita-wrapper.sh /home/mani/mani-wrapper.sh +ENV INST_IMG ${INST_IMG} +ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"] diff --git a/util/manibuilder/Dockerfile.centos b/util/manibuilder/Dockerfile.centos new file mode 100644 index 0000000..fcae9c8 --- /dev/null +++ b/util/manibuilder/Dockerfile.centos @@ -0,0 +1,16 @@ +FROM manibase + +RUN \ + useradd -p locked -m mani && \ + yum install -q -y git gcc pciutils-devel libusb-devel libusbx-devel && \ + yum clean -q -y all + +USER mani +RUN \ + cd && \ + mkdir .ccache && chown mani:mani .ccache && \ + git clone https://review.coreboot.org/flashrom.git + +ENV DEVSHELL /bin/bash +COPY mani-wrapper.sh /home/mani/ +ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"] diff --git a/util/manibuilder/Dockerfile.debian-debootstrap b/util/manibuilder/Dockerfile.debian-debootstrap new file mode 100644 index 0000000..c6648e8 --- /dev/null +++ b/util/manibuilder/Dockerfile.debian-debootstrap @@ -0,0 +1,20 @@ +FROM manibase + +RUN \ + useradd -p locked -m mani && \ + apt-get -qq update && \ + apt-get -qq upgrade && \ + apt-get -qq dist-upgrade && \ + apt-get -qqy install gcc make git doxygen ccache \ + libpci-dev libusb-dev libftdi-dev libusb-1.0-0-dev && \ + apt-get clean + +USER mani +RUN \ + cd && \ + mkdir .ccache && chown mani:mani .ccache && \ + git clone https://review.coreboot.org/flashrom.git + +ENV DEVSHELL /bin/bash +COPY mani-wrapper.sh /home/mani/ +ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"] diff --git a/util/manibuilder/Dockerfile.fedora b/util/manibuilder/Dockerfile.fedora new file mode 100644 index 0000000..927e487 --- /dev/null +++ b/util/manibuilder/Dockerfile.fedora @@ -0,0 +1,17 @@ +FROM manibase + +RUN \ + useradd -p locked -m mani && \ + dnf install -q -y git gcc ccache make \ + pciutils-devel libusb-devel libusbx-devel libftdi-devel && \ + dnf clean -q -y all + +USER mani +RUN \ + cd && \ + mkdir .ccache && chown mani:mani .ccache && \ + git clone https://review.coreboot.org/flashrom.git + +ENV DEVSHELL /bin/bash +COPY mani-wrapper.sh /home/mani/ +ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"] diff --git a/util/manibuilder/Dockerfile.qemu-user-static b/util/manibuilder/Dockerfile.qemu-user-static new file mode 100644 index 0000000..b6de7eb --- /dev/null +++ b/util/manibuilder/Dockerfile.qemu-user-static @@ -0,0 +1,3 @@ +FROM multiarch/qemu-user-static:register + +RUN sed -i -e's/ mipsn32 mipsn32el / /' /qemu-binfmt-conf.sh diff --git a/util/manibuilder/Dockerfile.ubuntu-debootstrap b/util/manibuilder/Dockerfile.ubuntu-debootstrap new file mode 100644 index 0000000..f50047c --- /dev/null +++ b/util/manibuilder/Dockerfile.ubuntu-debootstrap @@ -0,0 +1,33 @@ +FROM manibase + +RUN \ + useradd -p locked -m mani && \ + if grep -q main /etc/apt/sources.list; then \ + if ! grep -q universe /etc/apt/sources.list; then \ + sed -i -e 's/ main$/ main universe/' \ + /etc/apt/sources.list || exit 1; \ + fi; \ + else \ + url="http://ports.ubuntu.com/" && \ + cn="$(sed -ne's/DISTRIB_CODENAME=//p' /etc/lsb-release)" && \ + for t in "" "-updates" "-security"; do \ + echo "deb ${url} ${cn}${t} main universe" \ + >>/etc/apt/sources.list || exit 1; \ + done; \ + fi && \ + apt-get -qq update && \ + apt-get -qq upgrade && \ + apt-get -qq dist-upgrade && \ + apt-get -qqy install gcc make git doxygen ccache \ + libpci-dev libusb-dev libftdi-dev libusb-1.0-0-dev && \ + apt-get clean + +USER mani +RUN \ + cd && \ + mkdir .ccache && chown mani:mani .ccache && \ + git clone https://review.coreboot.org/flashrom.git + +ENV DEVSHELL /bin/bash +COPY mani-wrapper.sh /home/mani/ +ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"] diff --git a/util/manibuilder/Makefile b/util/manibuilder/Makefile new file mode 100644 index 0000000..a5cdeef --- /dev/null +++ b/util/manibuilder/Makefile @@ -0,0 +1,87 @@ +QUIET_TEST := @ + +include Makefile.targets + +CC := ccache cc +MAKECMD := make + +spc := +spc += + +stem = $(word 1,$(subst :,$(spc),$(subst :,$(spc),$(1)))) +ident = $(subst :,_,$(subst :,_,$(1))) + +PKGSRC_MIRROR = http://cdn.netbsd.org/ +NETBSD_MIRROR = http://ftp.de.netbsd.org/ + +anita:amd64-build: PKGSRC_PATH=pub/pkgsrc/packages/NetBSD/amd64/7.1/All +anita:amd64-build: NETBSD_IMAGE=pub/NetBSD/NetBSD-7.1/amd64/ + +anita:i386-build: PKGSRC_PATH=pub/pkgsrc/packages/NetBSD/i386/7.1/All +anita:i386-build: NETBSD_IMAGE=pub/NetBSD/NetBSD-7.1/i386/ + +$(addsuffix -build,$(ANITA_TAGS)): %-build: Dockerfile.anita anita-wrapper.sh + $(QUIET_SETUP)docker build . -f $< -t mani/$* \ + --build-arg PKG_PATH=$(PKGSRC_MIRROR)$(PKGSRC_PATH) \ + --build-arg INST_IMG=$(NETBSD_MIRROR)$(NETBSD_IMAGE) + +define build_template +Dockerfile.$(call ident,$(1)): Dockerfile.$(call stem,$(1)) mani-wrapper.sh + $(QUIET_SETUP)sed -e 's|^FROM manibase|FROM $(2)/$(1)|' $$< >$$@ + +.INTERMEDIATE: Dockerfile.$(call ident,$(1)) + +$(1)-build: Dockerfile.$(call ident,$(1)) + $(QUIET_SETUP)docker build . -f $$< -t mani/$(1) +endef + +$(foreach tag,$(MULTIARCH_TAGS), \ + $(eval $(call build_template,$(tag),multiarch))) + +$(addsuffix -check-build,$(ALL_TAGS)): %-check-build: + $(QUIET_SETUP)\ + [ $$(docker image ls -q mani/$*) ] \ + || $(MAKE) $*-build $(if $(QUIET_SETUP),>/dev/null 2>/dev/null) + +$(filter centos%,$(MULTIARCH_TAGS)) anita:sparc: CC=cc +$(ANITA_TAGS): MAKECMD=gmake +$(ALL_TAGS): export QUIET_SETUP=$(QUIET_TEST) +$(ALL_TAGS): %: %-check-build + $(QUIET_TEST)docker rm -f mani_$(call ident,$*) >/dev/null 2>&1 || true + $(QUIET_TEST)\ + docker run \ + --env IDENT=$(call ident,$*) \ + --volume manicache:/home/mani/.ccache \ + --name mani_$(call ident,$*) mani/$* \ + "git fetch origin $${TEST_REVISION:-master} && \ + git checkout FETCH_HEAD && \ + $(MAKECMD) clean && $(MAKECMD) -j$${CPUS:-1} CC='$(CC)'" \ + $(if $(QUIET_TEST),>/dev/null 2>&1) || echo $*: $$? + +$(addsuffix -shell,$(ALL_TAGS)): %-shell: %-check-build + $(QUIET_SETUP)\ + if [ $$(docker ps -a -q -f name=mani_$(call ident,$*)) ]; then \ + docker commit mani_$(call ident,$*) mani_run/$* && \ + docker run --rm -it \ + --env IDENT=$(call ident,$*) \ + --volume manicache:/home/mani/.ccache \ + --entrypoint /bin/sh mani_run/$* \ + /home/mani/mani-wrapper.sh \ + $(patsubst %,"%",$(SHELL_ARG)); \ + docker image rm mani_run/$*; \ + else \ + docker run --rm -it \ + --env IDENT=$(call ident,$*) \ + --volume manicache:/home/mani/.ccache \ + mani/$* $(patsubst %,"%",$(SHELL_ARG)); \ + fi + +.PHONY: $(foreach s,-build -check-build -shell, $(addsuffix $(s),$(ALL_TAGS))) + +register: + docker build . \ + -f Dockerfile.qemu-user-static \ + -t mani/qemu-user-static:register + docker run --rm --privileged mani/qemu-user-static:register --reset + +.PHONY: register diff --git a/util/manibuilder/Makefile.targets b/util/manibuilder/Makefile.targets new file mode 100644 index 0000000..a2cedce --- /dev/null +++ b/util/manibuilder/Makefile.targets @@ -0,0 +1,81 @@ +ANITA_TAGS := anita:amd64 anita:i386 anita:sparc + +MULTIARCH_TAGS := \ + $(foreach a,x86_64 ppc64le ppc64 armhfp aarch64, \ + $(foreach v,25 24, \ + fedora:$(v)-$(a))) \ + $(foreach a,aarch64 armhfp amd64, \ + $(foreach v,7.3 7.2, \ + centos:$(v)-$(a)-clean)) \ + $(foreach a,ppc64el powerpc arm64 armhf mips mipsel amd64 i386, \ + $(foreach v,sid stretch, \ + debian-debootstrap:$(a)-$(v))) \ + $(foreach a,ppc64el powerpc arm64 armhf amd64 i386, \ + $(foreach v,zesty xenial, \ + ubuntu-debootstrap:$(a)-$(v))) \ + $(foreach a,aarch64 armhf amd64 i386, \ + $(foreach v,v3.7 v3.6, \ + alpine:$(a)-$(v))) \ + +ALL_TAGS := $(ANITA_TAGS) $(MULTIARCH_TAGS) + +arch_filter = $(sort \ + $(foreach arch,$(1), \ + $(filter-out $(subst $(arch),,$(MULTIARCH_TAGS)),$(MULTIARCH_TAGS)))) + +machine_map = \ + $(if $(filter i386 i686 x86,$(1)),i386 x86, \ + $(if $(filter x86_64,$(1)),amd64 i386 x86, \ + $(if $(filter armv7l armv6l,$(1)),armhf, \ + $(if $(filter aarch64,$(1)),aarch64 arm64, \ + $(if $(filter ppc64le,$(1)),ppc64le ppc64el, \ + $(if $(filter ppc,$(1)),powerpc, \ + $(if $(filter mips,$(1)),mips mipsel, \ + $(1)))))))) + +NATIVE_TAGS := $(call arch_filter,$(call machine_map,$(shell uname -m))) + +# rather arbitrary selection of images that seem to work (focus on amd64) +DEFAULT_TAGS := \ + anita:amd64 \ + anita:i386 \ + fedora:25-x86_64 \ + fedora:25-ppc64le \ + fedora:25-aarch64 \ + fedora:24-x86_64 \ + centos:7.3-aarch64-clean \ + centos:7.3-amd64-clean \ + centos:7.2-amd64-clean \ + debian-debootstrap:ppc64el-stretch \ + debian-debootstrap:armhf-stretch \ + debian-debootstrap:mips-stretch \ + debian-debootstrap:mipsel-stretch \ + debian-debootstrap:amd64-stretch \ + debian-debootstrap:i386-stretch \ + debian-debootstrap:powerpc-sid \ + debian-debootstrap:amd64-sid \ + ubuntu-debootstrap:arm64-xenial \ + ubuntu-debootstrap:amd64-xenial \ + ubuntu-debootstrap:amd64-zesty \ + alpine:amd64-v3.6 \ + alpine:amd64-v3.7 \ + +# also run all native tests by default +DEFAULT_TAGS += $(filter-out $(DEFAULT_TAGS),$(NATIVE_TAGS)) + +default: $(DEFAULT_TAGS) + +native: $(NATIVE_TAGS) + +all: $(ALL_TAGS) + +show-default: + @echo $(DEFAULT_TAGS) + +show-native: + @echo $(NATIVE_TAGS) + +show-all: + @echo $(ALL_TAGS) + +.PHONY: default native all show-default show-native show-all $(ALL_TAGS) diff --git a/util/manibuilder/anita-wrapper.sh b/util/manibuilder/anita-wrapper.sh new file mode 100644 index 0000000..8f16888 --- /dev/null +++ b/util/manibuilder/anita-wrapper.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +cd + +[ "${IDENT}" ] || IDENT=$(mktemp -u XXXXXXXX) + +CCACHE=.ccache/anita-${IDENT}.img + +[ -f ${CCACHE} ] || zcat cache.img.gz >${CCACHE} + +if [ $# -eq 0 ]; then + exec anita --vmm-args "-hdb ${CCACHE}" interact ${INST_IMG} +else + exec anita --vmm-args "-hdb ${CCACHE}" --persist --run \ + "cd && . ./init && cd flashrom && $*; cd && umount .ccache" \ + boot ${INST_IMG} +fi diff --git a/util/manibuilder/mani-wrapper.sh b/util/manibuilder/mani-wrapper.sh new file mode 100644 index 0000000..c3f5834 --- /dev/null +++ b/util/manibuilder/mani-wrapper.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +cd /home/mani/flashrom/ + +if [ $# -eq 0 ]; then + exec "${DEVSHELL}" +else + exec "${DEVSHELL}" -c "$*" +fi