Nico Huber has uploaded this change for review.
[WIP] Add manibuilder
Add a set of Dockerfiles for build testing. If you have an x86 machine
and ~20GiB free disk space, run `make register` and `make -jxx` in
util/manibuilder and go eat some pizza. The former runs a privileged
docker container to set binfmt_misc up for qemu (read the code, don't
trust it).
Change-Id: I60863a5c7d70dde71486fccb66cb59b30ba4d982
Signed-off-by: Nico Huber <nico.h@gmx.de>
---
A util/manibuilder/Dockerfile.alpine
A util/manibuilder/Dockerfile.anita
A util/manibuilder/Dockerfile.centos
A util/manibuilder/Dockerfile.debian-debootstrap
A util/manibuilder/Dockerfile.fedora
A util/manibuilder/Dockerfile.qemu-user-static
A util/manibuilder/Dockerfile.ubuntu-debootstrap
A util/manibuilder/Makefile
A util/manibuilder/Makefile.targets
A util/manibuilder/anita-wrapper.sh
A util/manibuilder/mani-wrapper.sh
11 files changed, 341 insertions(+), 0 deletions(-)
git pull ssh://review.coreboot.org:29418/flashrom refs/changes/05/23005/1
diff --git a/util/manibuilder/Dockerfile.alpine b/util/manibuilder/Dockerfile.alpine
new file mode 100644
index 0000000..63d4ba3
--- /dev/null
+++ b/util/manibuilder/Dockerfile.alpine
@@ -0,0 +1,17 @@
+FROM manibase
+
+RUN \
+ adduser -D mani mani && \
+ apk update && \
+ apk add build-base linux-headers git ccache \
+ pciutils-dev libusb-compat-dev libusb-dev
+
+USER mani
+RUN \
+ cd && \
+ mkdir .ccache && chown mani:mani .ccache && \
+ git clone https://review.coreboot.org/flashrom.git
+
+ENV DEVSHELL /bin/sh
+COPY mani-wrapper.sh /home/mani/
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]
diff --git a/util/manibuilder/Dockerfile.anita b/util/manibuilder/Dockerfile.anita
new file mode 100644
index 0000000..71f830d
--- /dev/null
+++ b/util/manibuilder/Dockerfile.anita
@@ -0,0 +1,41 @@
+FROM debian:stable
+
+ARG PKG_PATH=http://cdn.netbsd.org/pub/pkgsrc/packages/NetBSD/amd64/7.1/All
+ARG INST_IMG=http://ftp.de.netbsd.org/pub/NetBSD/NetBSD-7.1/amd64/
+ARG EXTRA_PKG="ccache libftdi"
+
+RUN \
+ useradd -p locked -m mani && \
+ apt-get -qq update && \
+ apt-get -qq upgrade && \
+ apt-get -qq dist-upgrade && \
+ apt-get -qqy install git python python-pexpect \
+ genisoimage qemu-system && \
+ apt-get clean && \
+ git clone https://github.com/gson1703/anita.git && \
+ cd anita && python setup.py install
+
+USER mani
+RUN cd && mkdir .ccache && chown mani:mani .ccache && \
+ anita --sets kern-GENERIC,modules,base,etc,comp \
+ --disk-size 1024M --memory-size=128M install ${INST_IMG} && \
+ rm -rf work-*/download && \
+ dd if=/dev/zero bs=1M count=64 of=cache.img && \
+ anita --vmm-args '-hdb cache.img' --persist --run \
+"echo fdisk -0 -s 169/63/131009 -f -u wd1 && \
+ newfs wd1a && \
+ mkdir .ccache && \
+ echo 'mount /dev/wd1a .ccache' >init && \
+ echo dhcpcd >>init && \
+ echo export PKG_PATH=${PKG_PATH} >>init && . ./init && \
+ pkg_add gmake git-base pciutils libusb libusb1 ${EXTRA_PKG} && \
+ git config --global --add http.sslVerify false && \
+ git clone https://review.coreboot.org/flashrom.git && \
+ ccache -M 60M; \
+ umount .ccache" \
+ boot ${INST_IMG} && \
+ gzip cache.img
+
+COPY anita-wrapper.sh /home/mani/mani-wrapper.sh
+ENV INST_IMG ${INST_IMG}
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]
diff --git a/util/manibuilder/Dockerfile.centos b/util/manibuilder/Dockerfile.centos
new file mode 100644
index 0000000..fcae9c8
--- /dev/null
+++ b/util/manibuilder/Dockerfile.centos
@@ -0,0 +1,16 @@
+FROM manibase
+
+RUN \
+ useradd -p locked -m mani && \
+ yum install -q -y git gcc pciutils-devel libusb-devel libusbx-devel && \
+ yum clean -q -y all
+
+USER mani
+RUN \
+ cd && \
+ mkdir .ccache && chown mani:mani .ccache && \
+ git clone https://review.coreboot.org/flashrom.git
+
+ENV DEVSHELL /bin/bash
+COPY mani-wrapper.sh /home/mani/
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]
diff --git a/util/manibuilder/Dockerfile.debian-debootstrap b/util/manibuilder/Dockerfile.debian-debootstrap
new file mode 100644
index 0000000..c6648e8
--- /dev/null
+++ b/util/manibuilder/Dockerfile.debian-debootstrap
@@ -0,0 +1,20 @@
+FROM manibase
+
+RUN \
+ useradd -p locked -m mani && \
+ apt-get -qq update && \
+ apt-get -qq upgrade && \
+ apt-get -qq dist-upgrade && \
+ apt-get -qqy install gcc make git doxygen ccache \
+ libpci-dev libusb-dev libftdi-dev libusb-1.0-0-dev && \
+ apt-get clean
+
+USER mani
+RUN \
+ cd && \
+ mkdir .ccache && chown mani:mani .ccache && \
+ git clone https://review.coreboot.org/flashrom.git
+
+ENV DEVSHELL /bin/bash
+COPY mani-wrapper.sh /home/mani/
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]
diff --git a/util/manibuilder/Dockerfile.fedora b/util/manibuilder/Dockerfile.fedora
new file mode 100644
index 0000000..927e487
--- /dev/null
+++ b/util/manibuilder/Dockerfile.fedora
@@ -0,0 +1,17 @@
+FROM manibase
+
+RUN \
+ useradd -p locked -m mani && \
+ dnf install -q -y git gcc ccache make \
+ pciutils-devel libusb-devel libusbx-devel libftdi-devel && \
+ dnf clean -q -y all
+
+USER mani
+RUN \
+ cd && \
+ mkdir .ccache && chown mani:mani .ccache && \
+ git clone https://review.coreboot.org/flashrom.git
+
+ENV DEVSHELL /bin/bash
+COPY mani-wrapper.sh /home/mani/
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]
diff --git a/util/manibuilder/Dockerfile.qemu-user-static b/util/manibuilder/Dockerfile.qemu-user-static
new file mode 100644
index 0000000..b6de7eb
--- /dev/null
+++ b/util/manibuilder/Dockerfile.qemu-user-static
@@ -0,0 +1,3 @@
+FROM multiarch/qemu-user-static:register
+
+RUN sed -i -e's/ mipsn32 mipsn32el / /' /qemu-binfmt-conf.sh
diff --git a/util/manibuilder/Dockerfile.ubuntu-debootstrap b/util/manibuilder/Dockerfile.ubuntu-debootstrap
new file mode 100644
index 0000000..f50047c
--- /dev/null
+++ b/util/manibuilder/Dockerfile.ubuntu-debootstrap
@@ -0,0 +1,33 @@
+FROM manibase
+
+RUN \
+ useradd -p locked -m mani && \
+ if grep -q main /etc/apt/sources.list; then \
+ if ! grep -q universe /etc/apt/sources.list; then \
+ sed -i -e 's/ main$/ main universe/' \
+ /etc/apt/sources.list || exit 1; \
+ fi; \
+ else \
+ url="http://ports.ubuntu.com/" && \
+ cn="$(sed -ne's/DISTRIB_CODENAME=//p' /etc/lsb-release)" && \
+ for t in "" "-updates" "-security"; do \
+ echo "deb ${url} ${cn}${t} main universe" \
+ >>/etc/apt/sources.list || exit 1; \
+ done; \
+ fi && \
+ apt-get -qq update && \
+ apt-get -qq upgrade && \
+ apt-get -qq dist-upgrade && \
+ apt-get -qqy install gcc make git doxygen ccache \
+ libpci-dev libusb-dev libftdi-dev libusb-1.0-0-dev && \
+ apt-get clean
+
+USER mani
+RUN \
+ cd && \
+ mkdir .ccache && chown mani:mani .ccache && \
+ git clone https://review.coreboot.org/flashrom.git
+
+ENV DEVSHELL /bin/bash
+COPY mani-wrapper.sh /home/mani/
+ENTRYPOINT ["/bin/sh", "/home/mani/mani-wrapper.sh"]
diff --git a/util/manibuilder/Makefile b/util/manibuilder/Makefile
new file mode 100644
index 0000000..a5cdeef
--- /dev/null
+++ b/util/manibuilder/Makefile
@@ -0,0 +1,87 @@
+QUIET_TEST := @
+
+include Makefile.targets
+
+CC := ccache cc
+MAKECMD := make
+
+spc :=
+spc +=
+
+stem = $(word 1,$(subst :,$(spc),$(subst \:,$(spc),$(1))))
+ident = $(subst :,_,$(subst \:,_,$(1)))
+
+PKGSRC_MIRROR = http://cdn.netbsd.org/
+NETBSD_MIRROR = http://ftp.de.netbsd.org/
+
+anita\:amd64-build: PKGSRC_PATH=pub/pkgsrc/packages/NetBSD/amd64/7.1/All
+anita\:amd64-build: NETBSD_IMAGE=pub/NetBSD/NetBSD-7.1/amd64/
+
+anita\:i386-build: PKGSRC_PATH=pub/pkgsrc/packages/NetBSD/i386/7.1/All
+anita\:i386-build: NETBSD_IMAGE=pub/NetBSD/NetBSD-7.1/i386/
+
+$(addsuffix -build,$(ANITA_TAGS)): %-build: Dockerfile.anita anita-wrapper.sh
+ $(QUIET_SETUP)docker build . -f $< -t mani/$* \
+ --build-arg PKG_PATH=$(PKGSRC_MIRROR)$(PKGSRC_PATH) \
+ --build-arg INST_IMG=$(NETBSD_MIRROR)$(NETBSD_IMAGE)
+
+define build_template
+Dockerfile.$(call ident,$(1)): Dockerfile.$(call stem,$(1)) mani-wrapper.sh
+ $(QUIET_SETUP)sed -e 's|^FROM manibase|FROM $(2)/$(1)|' $$< >$$@
+
+.INTERMEDIATE: Dockerfile.$(call ident,$(1))
+
+$(1)-build: Dockerfile.$(call ident,$(1))
+ $(QUIET_SETUP)docker build . -f $$< -t mani/$(1)
+endef
+
+$(foreach tag,$(MULTIARCH_TAGS), \
+ $(eval $(call build_template,$(tag),multiarch)))
+
+$(addsuffix -check-build,$(ALL_TAGS)): %-check-build:
+ $(QUIET_SETUP)\
+ [ $$(docker image ls -q mani/$*) ] \
+ || $(MAKE) $*-build $(if $(QUIET_SETUP),>/dev/null 2>/dev/null)
+
+$(filter centos%,$(MULTIARCH_TAGS)) anita\:sparc: CC=cc
+$(ANITA_TAGS): MAKECMD=gmake
+$(ALL_TAGS): export QUIET_SETUP=$(QUIET_TEST)
+$(ALL_TAGS): %: %-check-build
+ $(QUIET_TEST)docker rm -f mani_$(call ident,$*) >/dev/null 2>&1 || true
+ $(QUIET_TEST)\
+ docker run \
+ --env IDENT=$(call ident,$*) \
+ --volume manicache:/home/mani/.ccache \
+ --name mani_$(call ident,$*) mani/$* \
+ "git fetch origin $${TEST_REVISION:-master} && \
+ git checkout FETCH_HEAD && \
+ $(MAKECMD) clean && $(MAKECMD) -j$${CPUS:-1} CC='$(CC)'" \
+ $(if $(QUIET_TEST),>/dev/null 2>&1) || echo $*: $$?
+
+$(addsuffix -shell,$(ALL_TAGS)): %-shell: %-check-build
+ $(QUIET_SETUP)\
+ if [ $$(docker ps -a -q -f name=mani_$(call ident,$*)) ]; then \
+ docker commit mani_$(call ident,$*) mani_run/$* && \
+ docker run --rm -it \
+ --env IDENT=$(call ident,$*) \
+ --volume manicache:/home/mani/.ccache \
+ --entrypoint /bin/sh mani_run/$* \
+ /home/mani/mani-wrapper.sh \
+ $(patsubst %,"%",$(SHELL_ARG)); \
+ docker image rm mani_run/$*; \
+ else \
+ docker run --rm -it \
+ --env IDENT=$(call ident,$*) \
+ --volume manicache:/home/mani/.ccache \
+ mani/$* $(patsubst %,"%",$(SHELL_ARG)); \
+ fi
+
+.PHONY: $(foreach s,-build -check-build -shell, $(addsuffix $(s),$(ALL_TAGS)))
+
+register:
+ docker build . \
+ -f Dockerfile.qemu-user-static \
+ -t mani/qemu-user-static:register
+ docker run --rm --privileged mani/qemu-user-static:register --reset
+
+.PHONY: register
diff --git a/util/manibuilder/Makefile.targets b/util/manibuilder/Makefile.targets
new file mode 100644
index 0000000..a2cedce
--- /dev/null
+++ b/util/manibuilder/Makefile.targets
@@ -0,0 +1,81 @@
+ANITA_TAGS := anita\:amd64 anita\:i386 anita\:sparc
+
+MULTIARCH_TAGS := \
+ $(foreach a,x86_64 ppc64le ppc64 armhfp aarch64, \
+ $(foreach v,25 24, \
+ fedora\:$(v)-$(a))) \
+ $(foreach a,aarch64 armhfp amd64, \
+ $(foreach v,7.3 7.2, \
+ centos\:$(v)-$(a)-clean)) \
+ $(foreach a,ppc64el powerpc arm64 armhf mips mipsel amd64 i386, \
+ $(foreach v,sid stretch, \
+ debian-debootstrap\:$(a)-$(v))) \
+ $(foreach a,ppc64el powerpc arm64 armhf amd64 i386, \
+ $(foreach v,zesty xenial, \
+ ubuntu-debootstrap\:$(a)-$(v))) \
+ $(foreach a,aarch64 armhf amd64 i386, \
+ $(foreach v,v3.7 v3.6, \
+ alpine\:$(a)-$(v))) \
+
+ALL_TAGS := $(ANITA_TAGS) $(MULTIARCH_TAGS)
+
+arch_filter = $(sort \
+ $(foreach arch,$(1), \
+ $(filter-out $(subst $(arch),,$(MULTIARCH_TAGS)),$(MULTIARCH_TAGS))))
+
+machine_map = \
+ $(if $(filter i386 i686 x86,$(1)),i386 x86, \
+ $(if $(filter x86_64,$(1)),amd64 i386 x86, \
+ $(if $(filter armv7l armv6l,$(1)),armhf, \
+ $(if $(filter aarch64,$(1)),aarch64 arm64, \
+ $(if $(filter ppc64le,$(1)),ppc64le ppc64el, \
+ $(if $(filter ppc,$(1)),powerpc, \
+ $(if $(filter mips,$(1)),mips mipsel, \
+ $(1))))))))
+
+NATIVE_TAGS := $(call arch_filter,$(call machine_map,$(shell uname -m)))
+
+# rather arbitrary selection of images that seem to work (focus on amd64)
+DEFAULT_TAGS := \
+ anita\:amd64 \
+ anita\:i386 \
+ fedora\:25-x86_64 \
+ fedora\:25-ppc64le \
+ fedora\:25-aarch64 \
+ fedora\:24-x86_64 \
+ centos\:7.3-aarch64-clean \
+ centos\:7.3-amd64-clean \
+ centos\:7.2-amd64-clean \
+ debian-debootstrap\:ppc64el-stretch \
+ debian-debootstrap\:armhf-stretch \
+ debian-debootstrap\:mips-stretch \
+ debian-debootstrap\:mipsel-stretch \
+ debian-debootstrap\:amd64-stretch \
+ debian-debootstrap\:i386-stretch \
+ debian-debootstrap\:powerpc-sid \
+ debian-debootstrap\:amd64-sid \
+ ubuntu-debootstrap\:arm64-xenial \
+ ubuntu-debootstrap\:amd64-xenial \
+ ubuntu-debootstrap\:amd64-zesty \
+ alpine\:amd64-v3.6 \
+ alpine\:amd64-v3.7 \
+
+# also run all native tests by default
+DEFAULT_TAGS += $(filter-out $(DEFAULT_TAGS),$(NATIVE_TAGS))
+
+default: $(DEFAULT_TAGS)
+
+native: $(NATIVE_TAGS)
+
+all: $(ALL_TAGS)
+
+show-default:
+ @echo $(DEFAULT_TAGS)
+
+show-native:
+ @echo $(NATIVE_TAGS)
+
+show-all:
+ @echo $(ALL_TAGS)
+
+.PHONY: default native all show-default show-native show-all $(ALL_TAGS)
diff --git a/util/manibuilder/anita-wrapper.sh b/util/manibuilder/anita-wrapper.sh
new file mode 100644
index 0000000..8f16888
--- /dev/null
+++ b/util/manibuilder/anita-wrapper.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+cd
+
+[ "${IDENT}" ] || IDENT=$(mktemp -u XXXXXXXX)
+
+CCACHE=.ccache/anita-${IDENT}.img
+
+[ -f ${CCACHE} ] || zcat cache.img.gz >${CCACHE}
+
+if [ $# -eq 0 ]; then
+ exec anita --vmm-args "-hdb ${CCACHE}" interact ${INST_IMG}
+else
+ exec anita --vmm-args "-hdb ${CCACHE}" --persist --run \
+ "cd && . ./init && cd flashrom && $*; cd && umount .ccache" \
+ boot ${INST_IMG}
+fi
diff --git a/util/manibuilder/mani-wrapper.sh b/util/manibuilder/mani-wrapper.sh
new file mode 100644
index 0000000..c3f5834
--- /dev/null
+++ b/util/manibuilder/mani-wrapper.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+cd /home/mani/flashrom/
+
+if [ $# -eq 0 ]; then
+ exec "${DEVSHELL}"
+else
+ exec "${DEVSHELL}" -c "$*"
+fi
To view, visit change 23005. To unsubscribe, or for help writing mail filters, visit settings.