Patrick Georgi (pgeorgi@google.com) just uploaded a new patch set to gerrit, which you can find at https://review.coreboot.org/18099
-gerrit
commit e35a8a91abc7b716553978ffde5b4b03f77a053d Author: Patrick Georgi pgeorgi@chromium.org Date: Wed Jan 11 15:26:58 2017 +0100
util/cbfstool: Add cbfs-compression-tool
cbfs-compression-tool provides a way to benchmark the compression algorithms as used by cbfstool (and coreboot) and allows to pre-compress data for later consumption by cbfstool (once it supports the format).
For an impression, the benchmark's results on my machine:
measuring 'none' compressing 10485760 bytes to 10485760 took 0 seconds measuring 'LZMA' compressing 10485760 bytes to 1736 took 2 seconds measuring 'LZ4' compressing 10485760 bytes to 41880 took 0 seconds
And a possible use for external compression, parallel and non-parallel (60MB in 53 files compressed to 650KB on a machine with 40 threads):
$ time (ls -1 *.* |xargs -n 1 -P $(nproc) -I '{}' cbfs-compression-tool compress '{}' out/'{}' LZMA)
real 0m0.786s user 0m11.440s sys 0m0.044s
$ time (ls -1 *.* |xargs -n 1 -P 1 -I '{}' cbfs-compression-tool compress '{}' out/'{}' LZMA)
real 0m10.444s user 0m10.280s sys 0m0.064s
Change-Id: I40be087e85d09a895b1ed277270350ab65a4d6d4 Signed-off-by: Patrick Georgi pgeorgi@chromium.org --- util/cbfstool/Makefile | 7 +- util/cbfstool/Makefile.inc | 36 ++++++--- util/cbfstool/cbfs.h | 5 -- util/cbfstool/cbfs_image.c | 9 --- util/cbfstool/cbfscomptool.c | 186 +++++++++++++++++++++++++++++++++++++++++++ util/cbfstool/common.h | 12 +++ 6 files changed, 227 insertions(+), 28 deletions(-)
diff --git a/util/cbfstool/Makefile b/util/cbfstool/Makefile index d6249bc..caa8c7d 100644 --- a/util/cbfstool/Makefile +++ b/util/cbfstool/Makefile @@ -9,7 +9,7 @@ OBJCOPY ?= objcopy VBOOT_SOURCE ?= $(top)/3rdparty/vboot
.PHONY: all -all: cbfstool fmaptool rmodtool ifwitool +all: cbfstool fmaptool rmodtool ifwitool cbfs-compression-tool
cbfstool: $(objutil)/cbfstool/cbfstool
@@ -19,13 +19,16 @@ rmodtool: $(objutil)/cbfstool/rmodtool
ifwitool: $(objutil)/cbfstool/ifwitool
-.PHONY: clean cbfstool fmaptool rmodtool ifwitool +cbfs-compression-tool: $(objutil)/cbfstool/cbfs-compression-tool + +.PHONY: clean cbfstool fmaptool rmodtool ifwitool cbfs-compression-tool clean: $(RM) fmd_parser.c fmd_parser.h fmd_scanner.c fmd_scanner.h $(RM) $(objutil)/cbfstool/cbfstool $(cbfsobj) $(RM) $(objutil)/cbfstool/fmaptool $(fmapobj) $(RM) $(objutil)/cbfstool/rmodtool $(rmodobj) $(RM) $(objutil)/cbfstool/ifwitool $(ifwiobj) + $(RM) $(objutil)/cbfstool/cbfs-compression-tool $(cbfscompobj)
linux_trampoline.c: linux_trampoline.S rm -f linux_trampoline.c diff --git a/util/cbfstool/Makefile.inc b/util/cbfstool/Makefile.inc index 6882cba..950092f 100644 --- a/util/cbfstool/Makefile.inc +++ b/util/cbfstool/Makefile.inc @@ -1,7 +1,20 @@ +compressionobj := +compressionobj += compress.o +# LZ4 +compressionobj += lz4.o +compressionobj += lz4hc.o +compressionobj += lz4frame.o +compressionobj += xxhash.o +compressionobj += lz4_wrapper.o +# LZMA +compressionobj += lzma.o +compressionobj += LzFind.o +compressionobj += LzmaDec.o +compressionobj += LzmaEnc.o + cbfsobj := cbfsobj += cbfstool.o cbfsobj += common.o -cbfsobj += compress.o cbfsobj += cbfs_image.o cbfsobj += cbfs-mkstage.o cbfsobj += cbfs-mkpayload.o @@ -13,24 +26,13 @@ cbfsobj += partitioned_file.o # COMMONLIB cbfsobj += cbfs.o cbfsobj += fsp_relocate.o -cbfsobj += lz4_wrapper.o cbfsobj += mem_pool.o cbfsobj += region.o -# LZMA -cbfsobj += lzma.o -cbfsobj += LzFind.o -cbfsobj += LzmaDec.o -cbfsobj += LzmaEnc.o # CRYPTOLIB cbfsobj += 2sha_utility.o cbfsobj += 2sha1.o cbfsobj += 2sha256.o cbfsobj += 2sha512.o -# LZ4 -cbfsobj += lz4.o -cbfsobj += lz4hc.o -cbfsobj += lz4frame.o -cbfsobj += xxhash.o # FMAP cbfsobj += fmap.o cbfsobj += kv_pair.o @@ -38,6 +40,8 @@ cbfsobj += valstr.o # linux as payload cbfsobj += linux_trampoline.o cbfsobj += cbfs-payload-linux.o +# compression algorithms +cbfsobj += $(compressionobj)
fmapobj := fmapobj += fmaptool.o @@ -62,6 +66,10 @@ ifwiobj := ifwiobj += ifwitool.o ifwiobj += common.o
+cbfscompobj := +cbfscompobj += $(compressionobj) +cbfscompobj += cbfscomptool.o + TOOLCFLAGS ?= -Werror -Wall -Wextra TOOLCFLAGS += -Wcast-qual -Wmissing-prototypes -Wredundant-decls -Wshadow TOOLCFLAGS += -Wstrict-prototypes -Wwrite-strings @@ -141,6 +149,10 @@ $(objutil)/cbfstool/ifwitool: $(addprefix $(objutil)/cbfstool/,$(ifwiobj)) printf " HOSTCC $(subst $(objutil)/,,$(@)) (link)\n" $(HOSTCC) $(TOOLLDFLAGS) -o $@ $(addprefix $(objutil)/cbfstool/,$(ifwiobj))
+$(objutil)/cbfstool/cbfs-compression-tool: $(addprefix $(objutil)/cbfstool/,$(cbfscompobj)) + printf " HOSTCC $(subst $(objutil)/,,$(@)) (link)\n" + $(HOSTCC) $(TOOLLDFLAGS) -o $@ $(addprefix $(objutil)/cbfstool/,$(cbfscompobj)) + # Yacc source is superset of header $(objutil)/cbfstool/fmd.o: TOOLCFLAGS += -Wno-redundant-decls $(objutil)/cbfstool/fmd_parser.o: TOOLCFLAGS += -Wno-redundant-decls diff --git a/util/cbfstool/cbfs.h b/util/cbfstool/cbfs.h index f1ae09d..451ef9f 100644 --- a/util/cbfstool/cbfs.h +++ b/util/cbfstool/cbfs.h @@ -204,11 +204,6 @@ struct cbfs_payload { */ #define CBFS_COMPONENT_NULL 0xFFFFFFFF
-struct typedesc_t { - uint32_t type; - const char *name; -}; - static struct typedesc_t filetypes[] unused = { {CBFS_COMPONENT_BOOTBLOCK, "bootblock"}, {CBFS_COMPONENT_CBFSHEADER, "cbfs header"}, diff --git a/util/cbfstool/cbfs_image.c b/util/cbfstool/cbfs_image.c index c34ca83..e530224 100644 --- a/util/cbfstool/cbfs_image.c +++ b/util/cbfstool/cbfs_image.c @@ -50,15 +50,6 @@ * (old) cbfstool. */ #define CBFS_FILENAME_ALIGN (16)
-/* Type and format */ - -static const struct typedesc_t types_cbfs_compression[] = { - {CBFS_COMPRESS_NONE, "none"}, - {CBFS_COMPRESS_LZMA, "LZMA"}, - {CBFS_COMPRESS_LZ4, "LZ4"}, - {0, NULL}, -}; - static const char *lookup_name_by_type(const struct typedesc_t *desc, uint32_t type, const char *default_value) { diff --git a/util/cbfstool/cbfscomptool.c b/util/cbfstool/cbfscomptool.c new file mode 100644 index 0000000..dfe4d4b --- /dev/null +++ b/util/cbfstool/cbfscomptool.c @@ -0,0 +1,186 @@ +/* + * cbfs-compression-tool, CLI utility for dealing with CBFS compressed data + * + * Copyright (C) 2017 Google, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +#include "common.h" + +void usage(void); +int benchmark(void); +int compress(char *infile, char *outfile, char *algoname); + +const char *usage_text = "cbfs-compression-tool benchmark\n" + " runs benchmarks for all implemented algorithms\n" + "cbfs-compression-tool compress inFile outFile algo\n" + " compresses inFile with algo and stores in outFile\n" + "\n" + "'compress' file format:\n" + " 4 bytes little endian: algorithm ID (as used in CBFS)\n" + " 4 bytes little endian: uncompressed size\n" + " ...: compressed data stream\n"; + +void usage() +{ + puts(usage_text); +} + +int benchmark() +{ + const int bufsize = 10*1024*1024; + char *data = malloc(bufsize); + if (!data) { + fprintf(stderr, "out of memory\n"); + return 1; + } + char *compressed_data = malloc(bufsize); + if (!compressed_data) { + fprintf(stderr, "out of memory\n"); + return 1; + } + int i, l = strlen(usage_text) + 1; + for (i = 0; i + l < bufsize; i += l) { + memcpy(data + i, usage_text, l); + } + memset(data + i, 0, bufsize - i); + const struct typedesc_t *algo; + for (algo = &types_cbfs_compression[0]; algo->name != NULL; algo++) { + int outsize = bufsize; + printf("measuring '%s'\n", algo->name); + comp_func_ptr comp = compression_function(algo->type); + if (comp == NULL) { + printf("no handler associated with algorithm\n"); + return 1; + } + + struct timespec t_s, t_e; + clock_gettime(CLOCK_MONOTONIC, &t_s); + + if (comp(data, bufsize, compressed_data, &outsize)) { + printf("compression failed"); + return 1; + } + + clock_gettime(CLOCK_MONOTONIC, &t_e); + printf("compressing %d bytes to %d took %ld seconds\n", + bufsize, outsize, + t_e.tv_sec - t_s.tv_sec); + } + return 0; +} + +int compress(char *infile, char *outfile, char *algoname) +{ + int err = 1; + + const struct typedesc_t *algo = &types_cbfs_compression[0]; + while (algo->name != NULL) { + if (strcmp(algo->name, algoname) == 0) break; + algo++; + } + if (algo->name == NULL) { + fprintf(stderr, "algo '%s' is not supported.\n", algoname); + } + + comp_func_ptr comp = compression_function(algo->type); + if (comp == NULL) { + printf("no handler associated with algorithm\n"); + return 1; + } + + FILE *fin = fopen(infile, "rb"); + if (!fin) { + fprintf(stderr, "could not open '%s'\n", infile); + return 1; + } + FILE *fout = fopen(outfile, "wb"); + if (!fout) { + fprintf(stderr, "could not open '%s' for writing\n", outfile); + goto out; + } + + if (fseek(fin, 0, SEEK_END) != 0) { + fprintf(stderr, "could not seek in input\n"); + goto out; + } + long insize = ftell(fin); + if (insize < 0) { + fprintf(stderr, "could not determine input size\n"); + goto out; + } + rewind(fin); + + void *indata = malloc(insize); + if (!indata) { + fprintf(stderr, "out of memory\n"); + goto out; + } + + void *outdata = malloc(insize); + if (!outdata) { + fprintf(stderr, "out of memory\n"); + goto out; + } + int outsize; + + int remsize = insize; + while (remsize > 0) { + int readsz = fread(indata, 1, remsize, fin); + if (readsz < 0) { + fprintf(stderr, "failed to read input with %d bytes left\n", remsize); + goto out; + } + remsize -= readsz; + } + + comp(indata, insize, outdata, &outsize); + + char header[8]; + header[0] = algo->type & 0xff; + header[1] = (algo->type >> 8) & 0xff; + header[2] = (algo->type >> 16) & 0xff; + header[3] = (algo->type >> 24) & 0xff; + header[4] = insize & 0xff; + header[5] = (insize >> 8) & 0xff; + header[6] = (insize >> 16) & 0xff; + header[7] = (insize >> 24) & 0xff; + if (fwrite(header, 8, 1, fout) != 1) { + fprintf(stderr, "failed writing header\n"); + goto out; + } + if (fwrite(outdata, outsize, 1, fout) != 1) { + fprintf(stderr, "failed writing compressed data\n"); + goto out; + } + + err = 0; +out: + if (fin) fclose(fin); + if (fout) fclose(fout); + if (indata) free(indata); + return err; +} + +int main(int argc, char **argv) +{ + if ((argc == 2) && (strcmp(argv[1], "benchmark") == 0)) + return benchmark(); + if ((argc == 5) && (strcmp(argv[1], "compress") == 0)) + return compress(argv[2], argv[3], argv[4]); + usage(); + return 1; +} diff --git a/util/cbfstool/common.h b/util/cbfstool/common.h index d4b7021..85dfdeb 100644 --- a/util/cbfstool/common.h +++ b/util/cbfstool/common.h @@ -166,6 +166,18 @@ enum comp_algo { CBFS_COMPRESS_LZ4 = 2, };
+struct typedesc_t { + uint32_t type; + const char *name; +}; + +static const struct typedesc_t types_cbfs_compression[] = { + {CBFS_COMPRESS_NONE, "none"}, + {CBFS_COMPRESS_LZMA, "LZMA"}, + {CBFS_COMPRESS_LZ4, "LZ4"}, + {0, NULL}, +}; + comp_func_ptr compression_function(enum comp_algo algo); decomp_func_ptr decompression_function(enum comp_algo algo);