[coreboot-gerrit] Patch set updated for coreboot: util/cbfstool: Add cbfs-compression-tool
Patrick Georgi (pgeorgi@google.com)
gerrit at coreboot.org
Wed Jan 11 18:49:57 CET 2017
Patrick Georgi (pgeorgi at google.com) just uploaded a new patch set to gerrit, which you can find at https://review.coreboot.org/18099
-gerrit
commit ad5d78b843e4a0872da1a8175e86d43b36707d1d
Author: Patrick Georgi <pgeorgi at chromium.org>
Date: Wed Jan 11 15:26:58 2017 +0100
util/cbfstool: Add cbfs-compression-tool
cbfs-compression-tool provides a way to benchmark the compression
algorithms as used by cbfstool (and coreboot) and allows to
pre-compress data for later consumption by cbfstool (once it supports
the format).
For an impression, the benchmark's results on my machine:
measuring 'none'
compressing 10485760 bytes to 10485760 took 0 seconds
measuring 'LZMA'
compressing 10485760 bytes to 1736 took 2 seconds
measuring 'LZ4'
compressing 10485760 bytes to 41880 took 0 seconds
And a possible use for external compression, parallel and non-parallel
(60MB in 53 files compressed to 650KB on a machine with 40 threads):
$ time (ls -1 *.* |xargs -n 1 -P $(nproc) -I '{}' cbfs-compression-tool compress '{}' out/'{}' LZMA)
real 0m0.786s
user 0m11.440s
sys 0m0.044s
$ time (ls -1 *.* |xargs -n 1 -P 1 -I '{}' cbfs-compression-tool compress '{}' out/'{}' LZMA)
real 0m10.444s
user 0m10.280s
sys 0m0.064s
Change-Id: I40be087e85d09a895b1ed277270350ab65a4d6d4
Signed-off-by: Patrick Georgi <pgeorgi at chromium.org>
---
util/cbfstool/Makefile | 7 +-
util/cbfstool/Makefile.inc | 36 ++++++---
util/cbfstool/cbfs.h | 5 --
util/cbfstool/cbfs_image.c | 9 ---
util/cbfstool/cbfscomptool.c | 189 +++++++++++++++++++++++++++++++++++++++++++
util/cbfstool/common.h | 12 +++
6 files changed, 230 insertions(+), 28 deletions(-)
diff --git a/util/cbfstool/Makefile b/util/cbfstool/Makefile
index d6249bc..caa8c7d 100644
--- a/util/cbfstool/Makefile
+++ b/util/cbfstool/Makefile
@@ -9,7 +9,7 @@ OBJCOPY ?= objcopy
VBOOT_SOURCE ?= $(top)/3rdparty/vboot
.PHONY: all
-all: cbfstool fmaptool rmodtool ifwitool
+all: cbfstool fmaptool rmodtool ifwitool cbfs-compression-tool
cbfstool: $(objutil)/cbfstool/cbfstool
@@ -19,13 +19,16 @@ rmodtool: $(objutil)/cbfstool/rmodtool
ifwitool: $(objutil)/cbfstool/ifwitool
-.PHONY: clean cbfstool fmaptool rmodtool ifwitool
+cbfs-compression-tool: $(objutil)/cbfstool/cbfs-compression-tool
+
+.PHONY: clean cbfstool fmaptool rmodtool ifwitool cbfs-compression-tool
clean:
$(RM) fmd_parser.c fmd_parser.h fmd_scanner.c fmd_scanner.h
$(RM) $(objutil)/cbfstool/cbfstool $(cbfsobj)
$(RM) $(objutil)/cbfstool/fmaptool $(fmapobj)
$(RM) $(objutil)/cbfstool/rmodtool $(rmodobj)
$(RM) $(objutil)/cbfstool/ifwitool $(ifwiobj)
+ $(RM) $(objutil)/cbfstool/cbfs-compression-tool $(cbfscompobj)
linux_trampoline.c: linux_trampoline.S
rm -f linux_trampoline.c
diff --git a/util/cbfstool/Makefile.inc b/util/cbfstool/Makefile.inc
index 6882cba..950092f 100644
--- a/util/cbfstool/Makefile.inc
+++ b/util/cbfstool/Makefile.inc
@@ -1,7 +1,20 @@
+compressionobj :=
+compressionobj += compress.o
+# LZ4
+compressionobj += lz4.o
+compressionobj += lz4hc.o
+compressionobj += lz4frame.o
+compressionobj += xxhash.o
+compressionobj += lz4_wrapper.o
+# LZMA
+compressionobj += lzma.o
+compressionobj += LzFind.o
+compressionobj += LzmaDec.o
+compressionobj += LzmaEnc.o
+
cbfsobj :=
cbfsobj += cbfstool.o
cbfsobj += common.o
-cbfsobj += compress.o
cbfsobj += cbfs_image.o
cbfsobj += cbfs-mkstage.o
cbfsobj += cbfs-mkpayload.o
@@ -13,24 +26,13 @@ cbfsobj += partitioned_file.o
# COMMONLIB
cbfsobj += cbfs.o
cbfsobj += fsp_relocate.o
-cbfsobj += lz4_wrapper.o
cbfsobj += mem_pool.o
cbfsobj += region.o
-# LZMA
-cbfsobj += lzma.o
-cbfsobj += LzFind.o
-cbfsobj += LzmaDec.o
-cbfsobj += LzmaEnc.o
# CRYPTOLIB
cbfsobj += 2sha_utility.o
cbfsobj += 2sha1.o
cbfsobj += 2sha256.o
cbfsobj += 2sha512.o
-# LZ4
-cbfsobj += lz4.o
-cbfsobj += lz4hc.o
-cbfsobj += lz4frame.o
-cbfsobj += xxhash.o
# FMAP
cbfsobj += fmap.o
cbfsobj += kv_pair.o
@@ -38,6 +40,8 @@ cbfsobj += valstr.o
# linux as payload
cbfsobj += linux_trampoline.o
cbfsobj += cbfs-payload-linux.o
+# compression algorithms
+cbfsobj += $(compressionobj)
fmapobj :=
fmapobj += fmaptool.o
@@ -62,6 +66,10 @@ ifwiobj :=
ifwiobj += ifwitool.o
ifwiobj += common.o
+cbfscompobj :=
+cbfscompobj += $(compressionobj)
+cbfscompobj += cbfscomptool.o
+
TOOLCFLAGS ?= -Werror -Wall -Wextra
TOOLCFLAGS += -Wcast-qual -Wmissing-prototypes -Wredundant-decls -Wshadow
TOOLCFLAGS += -Wstrict-prototypes -Wwrite-strings
@@ -141,6 +149,10 @@ $(objutil)/cbfstool/ifwitool: $(addprefix $(objutil)/cbfstool/,$(ifwiobj))
printf " HOSTCC $(subst $(objutil)/,,$(@)) (link)\n"
$(HOSTCC) $(TOOLLDFLAGS) -o $@ $(addprefix $(objutil)/cbfstool/,$(ifwiobj))
+$(objutil)/cbfstool/cbfs-compression-tool: $(addprefix $(objutil)/cbfstool/,$(cbfscompobj))
+ printf " HOSTCC $(subst $(objutil)/,,$(@)) (link)\n"
+ $(HOSTCC) $(TOOLLDFLAGS) -o $@ $(addprefix $(objutil)/cbfstool/,$(cbfscompobj))
+
# Yacc source is superset of header
$(objutil)/cbfstool/fmd.o: TOOLCFLAGS += -Wno-redundant-decls
$(objutil)/cbfstool/fmd_parser.o: TOOLCFLAGS += -Wno-redundant-decls
diff --git a/util/cbfstool/cbfs.h b/util/cbfstool/cbfs.h
index f1ae09d..451ef9f 100644
--- a/util/cbfstool/cbfs.h
+++ b/util/cbfstool/cbfs.h
@@ -204,11 +204,6 @@ struct cbfs_payload {
*/
#define CBFS_COMPONENT_NULL 0xFFFFFFFF
-struct typedesc_t {
- uint32_t type;
- const char *name;
-};
-
static struct typedesc_t filetypes[] unused = {
{CBFS_COMPONENT_BOOTBLOCK, "bootblock"},
{CBFS_COMPONENT_CBFSHEADER, "cbfs header"},
diff --git a/util/cbfstool/cbfs_image.c b/util/cbfstool/cbfs_image.c
index c34ca83..e530224 100644
--- a/util/cbfstool/cbfs_image.c
+++ b/util/cbfstool/cbfs_image.c
@@ -50,15 +50,6 @@
* (old) cbfstool. */
#define CBFS_FILENAME_ALIGN (16)
-/* Type and format */
-
-static const struct typedesc_t types_cbfs_compression[] = {
- {CBFS_COMPRESS_NONE, "none"},
- {CBFS_COMPRESS_LZMA, "LZMA"},
- {CBFS_COMPRESS_LZ4, "LZ4"},
- {0, NULL},
-};
-
static const char *lookup_name_by_type(const struct typedesc_t *desc, uint32_t type,
const char *default_value)
{
diff --git a/util/cbfstool/cbfscomptool.c b/util/cbfstool/cbfscomptool.c
new file mode 100644
index 0000000..1aa1699
--- /dev/null
+++ b/util/cbfstool/cbfscomptool.c
@@ -0,0 +1,189 @@
+/*
+ * cbfs-compression-tool, CLI utility for dealing with CBFS compressed data
+ *
+ * Copyright (C) 2017 Google, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include "common.h"
+
+void usage(void);
+int benchmark(void);
+int compress(char *infile, char *outfile, char *algoname);
+
+const char *usage_text = "cbfs-compression-tool benchmark\n"
+ " runs benchmarks for all implemented algorithms\n"
+ "cbfs-compression-tool compress inFile outFile algo\n"
+ " compresses inFile with algo and stores in outFile\n"
+ "\n"
+ "'compress' file format:\n"
+ " 4 bytes little endian: algorithm ID (as used in CBFS)\n"
+ " 4 bytes little endian: uncompressed size\n"
+ " ...: compressed data stream\n";
+
+void usage()
+{
+ puts(usage_text);
+}
+
+int benchmark()
+{
+ const int bufsize = 10*1024*1024;
+ char *data = malloc(bufsize);
+ if (!data) {
+ fprintf(stderr, "out of memory\n");
+ return 1;
+ }
+ char *compressed_data = malloc(bufsize);
+ if (!compressed_data) {
+ fprintf(stderr, "out of memory\n");
+ return 1;
+ }
+ int i, l = strlen(usage_text) + 1;
+ for (i = 0; i + l < bufsize; i += l) {
+ memcpy(data + i, usage_text, l);
+ }
+ memset(data + i, 0, bufsize - i);
+ const struct typedesc_t *algo;
+ for (algo = &types_cbfs_compression[0]; algo->name != NULL; algo++) {
+ int outsize = bufsize;
+ printf("measuring '%s'\n", algo->name);
+ comp_func_ptr comp = compression_function(algo->type);
+ if (comp == NULL) {
+ printf("no handler associated with algorithm\n");
+ return 1;
+ }
+
+ struct timespec t_s, t_e;
+ clock_gettime(CLOCK_MONOTONIC, &t_s);
+
+ if (comp(data, bufsize, compressed_data, &outsize)) {
+ printf("compression failed");
+ return 1;
+ }
+
+ clock_gettime(CLOCK_MONOTONIC, &t_e);
+ printf("compressing %d bytes to %d took %ld seconds\n",
+ bufsize, outsize,
+ t_e.tv_sec - t_s.tv_sec);
+ }
+ return 0;
+}
+
+int compress(char *infile, char *outfile, char *algoname)
+{
+ int err = 1;
+ FILE *fin = NULL;
+ FILE *fout = NULL;
+ void *indata = NULL;
+
+ const struct typedesc_t *algo = &types_cbfs_compression[0];
+ while (algo->name != NULL) {
+ if (strcmp(algo->name, algoname) == 0) break;
+ algo++;
+ }
+ if (algo->name == NULL) {
+ fprintf(stderr, "algo '%s' is not supported.\n", algoname);
+ }
+
+ comp_func_ptr comp = compression_function(algo->type);
+ if (comp == NULL) {
+ printf("no handler associated with algorithm\n");
+ return 1;
+ }
+
+ fin = fopen(infile, "rb");
+ if (!fin) {
+ fprintf(stderr, "could not open '%s'\n", infile);
+ return 1;
+ }
+ fout = fopen(outfile, "wb");
+ if (!fout) {
+ fprintf(stderr, "could not open '%s' for writing\n", outfile);
+ goto out;
+ }
+
+ if (fseek(fin, 0, SEEK_END) != 0) {
+ fprintf(stderr, "could not seek in input\n");
+ goto out;
+ }
+ long insize = ftell(fin);
+ if (insize < 0) {
+ fprintf(stderr, "could not determine input size\n");
+ goto out;
+ }
+ rewind(fin);
+
+ indata = malloc(insize);
+ if (!indata) {
+ fprintf(stderr, "out of memory\n");
+ goto out;
+ }
+
+ void *outdata = malloc(insize);
+ if (!outdata) {
+ fprintf(stderr, "out of memory\n");
+ goto out;
+ }
+ int outsize;
+
+ int remsize = insize;
+ while (remsize > 0) {
+ int readsz = fread(indata, 1, remsize, fin);
+ if (readsz < 0) {
+ fprintf(stderr, "failed to read input with %d bytes left\n", remsize);
+ goto out;
+ }
+ remsize -= readsz;
+ }
+
+ comp(indata, insize, outdata, &outsize);
+
+ char header[8];
+ header[0] = algo->type & 0xff;
+ header[1] = (algo->type >> 8) & 0xff;
+ header[2] = (algo->type >> 16) & 0xff;
+ header[3] = (algo->type >> 24) & 0xff;
+ header[4] = insize & 0xff;
+ header[5] = (insize >> 8) & 0xff;
+ header[6] = (insize >> 16) & 0xff;
+ header[7] = (insize >> 24) & 0xff;
+ if (fwrite(header, 8, 1, fout) != 1) {
+ fprintf(stderr, "failed writing header\n");
+ goto out;
+ }
+ if (fwrite(outdata, outsize, 1, fout) != 1) {
+ fprintf(stderr, "failed writing compressed data\n");
+ goto out;
+ }
+
+ err = 0;
+out:
+ if (fin) fclose(fin);
+ if (fout) fclose(fout);
+ if (indata) free(indata);
+ return err;
+}
+
+int main(int argc, char **argv)
+{
+ if ((argc == 2) && (strcmp(argv[1], "benchmark") == 0))
+ return benchmark();
+ if ((argc == 5) && (strcmp(argv[1], "compress") == 0))
+ return compress(argv[2], argv[3], argv[4]);
+ usage();
+ return 1;
+}
diff --git a/util/cbfstool/common.h b/util/cbfstool/common.h
index d4b7021..85dfdeb 100644
--- a/util/cbfstool/common.h
+++ b/util/cbfstool/common.h
@@ -166,6 +166,18 @@ enum comp_algo {
CBFS_COMPRESS_LZ4 = 2,
};
+struct typedesc_t {
+ uint32_t type;
+ const char *name;
+};
+
+static const struct typedesc_t types_cbfs_compression[] = {
+ {CBFS_COMPRESS_NONE, "none"},
+ {CBFS_COMPRESS_LZMA, "LZMA"},
+ {CBFS_COMPRESS_LZ4, "LZ4"},
+ {0, NULL},
+};
+
comp_func_ptr compression_function(enum comp_algo algo);
decomp_func_ptr decompression_function(enum comp_algo algo);
More information about the coreboot-gerrit
mailing list