[LinuxBIOS] [PATCH][v3] enable compression support in lbv3's lar

[Patrick Georgi]

Hi,

I attach a patch to enable compression and decompression in lar.
After some discussion with Uwe, this patch cleans up some style issues 
(I hope I got them all, please bear with me if not!), presents some more 
information to the user and in a cleaner way, and also provides a way to 
mark files as "not compress", by prefixing their filename with "nocompress:"

Comments appreciated.

Regards,
Patrick Georgi

- extend lar format to support compression (incompatible format change!)
- adapt the compression utilities for integration into lar
- add compression capabilities to lar and expose in user interface
- provide a way to mark files as non-compressible

Signed-off-by: Patrick Georgi

Index: include/lar.h
===================================================================
--- include/lar.h	(revision 364)
+++ include/lar.h	(working copy)
@@ -58,8 +58,11 @@
 struct lar_header {
    char magic[8];
    u32 len;
+	u32 reallen;
    u32 checksum;
+	u32 compchecksum;
    u32 offset;
+	u32 compression;
 };

struct mem_file {
Index: util/lzma/minilzma.cc
===================================================================
--- util/lzma/minilzma.cc	(revision 364)
+++ util/lzma/minilzma.cc	(working copy)
@@ -281,13 +281,19 @@
 #else
 extern "C" {

-void do_lzma_compress(char* in, unsigned long in_len, char* out, unsigned long* out_len) {
+void do_lzma_compress(char *in, unsigned long in_len, char *out, unsigned long *out_len) {
    std::vector<unsigned char> result;
    result = LZMACompress(std::vector<unsigned char>(in,in+in_len));
    *out_len = result.size();
     	std::memcpy(out, &result[0], *out_len);
 }

+void do_lzma_uncompress(char *dst, char *src, unsigned long len) {
+	std::vector<unsigned char> result;
+	result = LZMADeCompress(std::vector<unsigned char>(src,src+len));
+	std::memcpy(dst, &result[0], result.size());
 }
+
+}
 #endif

Index: util/nrv2b/nrv2b.c
===================================================================
--- util/nrv2b/nrv2b.c	(revision 364)
+++ util/nrv2b/nrv2b.c	(working copy)
@@ -1304,6 +1304,60 @@
    out = malloc(*out_len);
    ucl_nrv2b_99_compress(in, in_len, out, out_len, 0 );
 }
+
+void do_nrv2b_uncompress(char* dst, char* src, unsigned long len) {
+	unsigned long ilen = 0, olen = 0, last_m_off =  1;
+	for (;;) {
+		unsigned int m_off, m_len;
+		while(GETBIT(bb, src, ilen)) {
+			FAIL(ilen >= src_len, "input overrun");
+			FAIL(olen >= dst_len, "output  overrun");
+			dst[olen++] = src[ilen++];
+		}
+		m_off = 1;
+		do {
+			m_off = m_off*2 + GETBIT(bb, src, ilen);
+			FAIL(ilen >= src_len, "input overrun");
+			FAIL(m_off > 0xffffffU +3, "lookbehind overrun");
+		} while (!GETBIT(bb, src, ilen));
+		if (m_off == 2)
+		{
+			m_off = last_m_off;
+		}
+		else
+		{
+			FAIL(ilen >= src_len, "input overrun");
+			m_off = (m_off - 3)*256 + src[ilen++];
+			if(m_off == 0xffffffffU)
+				break;
+			last_m_off = ++m_off;
+		}
+		m_len = GETBIT(bb, src, ilen);
+		m_len = m_len*2 + GETBIT(bb, src, ilen);
+		if (m_len == 0) 
+		{
+			m_len++;
+			do {
+				m_len = m_len*2 + GETBIT(bb, src, ilen);
+				FAIL(ilen >= src_len, "input overrun");
+				FAIL(m_len >= dst_len, "output overrun");
+			} while(!GETBIT(bb, src, ilen));
+			m_len += 2;
+		}
+		m_len += (m_off > 0xd00);
+		FAIL(olen + m_len > dst_len, "output overrun");
+		FAIL(m_off > olen, "lookbeind overrun");
+		{
+			const uint8_t *m_pos;
+			m_pos = dst + olen - m_off;
+			dst[olen++] = *m_pos++;
+			do {
+				dst[olen++] = *m_pos++;
+			} while(--m_len > 0);
+		}
+	}
+	FAIL(ilen < src_len, "input not consumed");
+}
 #endif

#ifdef DECODE
Index: util/lar/lar.c
===================================================================
--- util/lar/lar.c	(revision 364)
+++ util/lar/lar.c	(working copy)
@@ -39,6 +39,7 @@
 static int isverbose = 0;
 static long larsize = 0;
 static char *bootblock = NULL;
+enum compalgo algo = none;

static void usage(char *name)
 {
@@ -72,6 +73,7 @@

static struct option long_options[] = {
    	{"create", 0, 0, 'c'},
+		{"compress-algo", 1, 0, 'C'},
    	{"extract", 0, 0, 'x'},
    	{"list", 0, 0, 'l'},
    	{"size", 1, 0, 's'},
@@ -87,12 +89,20 @@
    	exit(1);
    }

-	while ((opt = getopt_long(argc, argv, "cxls:b:vVh?",
+	while ((opt = getopt_long(argc, argv, "cC:xls:b:vVh?",
    			  long_options, &option_index)) != EOF) {
    	switch (opt) {
    	case 'c':
    		larmode = CREATE;
    		break;
+		case 'C':
+			if (strcmp("lzma",optarg)==0) {
+				algo=lzma;
+			}
+			if (strcmp("nrv2b",optarg)==0) {
+				algo=nrv2b;
+			}
+			break;
    	case 'l':
    		larmode = LIST;
    		break;
Index: util/lar/create.c
===================================================================
--- util/lar/create.c	(revision 364)
+++ util/lar/create.c	(working copy)
@@ -34,6 +34,17 @@
 #include "lib.h"
 #include "lar.h"

+extern enum compalgo algo;
+void compress_impossible(char* in, u32 in_len, char* out, u32* out_len) {
+	fprintf(stderr,"the selected compression algorithm wasn't compiled in.\n");
+	exit(1);
+}
+void do_no_compress(char* in, u32 in_len, char* out, u32* out_len) {
+	memcpy(out,in,in_len);
+	out_len[0]=in_len;
+}
+
+
 int create_lar(const char *archivename, struct file *files)
 {
    int i, ret;
@@ -41,14 +52,16 @@
    int bb_header_len = 0;
    FILE *archive, *source;
    char *tempmem;
-	char *filebuf;
+	char *filebuf, *filetarget;
    char *pathname;
    u32 *walk;
    u32 csum;
    int pathlen, entrylen, filelen;
+	u32 compfilelen;
    long currentsize = 0;
    struct lar_header *header;
    struct stat statbuf;
+	enum compalgo thisalgo;

if (!files) {
    	fprintf(stderr, "No files for archive %s\n", archivename);
@@ -68,6 +81,12 @@
    while (files) {
    	char *name = files->name;

+		thisalgo = algo;
+
+		if (strstr(name,"nocompress:") == name) {
+			name += 11;
+			thisalgo = none;
+		}
    	/* skip ./ if available */
    	if (name[0] == '.' && name[1] == '/')
    		name += 2;
@@ -97,24 +116,33 @@
    	pathlen = (pathlen + 15) & 0xfffffff0;/* Align to 16 bytes. */

/* Read file into memory. */
-		filebuf = pathname + pathlen;
+		filebuf = malloc(filelen);
+		filetarget = pathname + pathlen;
    	source = fopen(name, "r");
    	if (!source) {
    		fprintf(stderr, "No such file %s\n", name);
    		exit(1);
    	}
-		fread(filebuf, statbuf.st_size, 1, source);
+		fread(filebuf, filelen, 1, source);
    	fclose(source);
+		compress_functions[thisalgo](filebuf, filelen, filetarget, &compfilelen);
+		if ((compfilelen >= filelen) && (thisalgo != none)) {
+			thisalgo = none;
+			compress_functions[thisalgo](filebuf, filelen, filetarget, &compfilelen);
+		}
+		free(filebuf);

/* Create correct header. */
    	memcpy(header, MAGIC, 8);
-		header->len = htonl(statbuf.st_size);
+		header->compression = htonl(thisalgo);
+		header->reallen = htonl(filelen);
+		header->len = htonl(compfilelen);
    	header->offset = htonl(sizeof(struct lar_header) + pathlen);

/* Calculate checksum. */
    	csum = 0;
    	for (walk = (u32 *) tempmem;
-		     walk < (u32 *) (tempmem + statbuf.st_size +
+		     walk < (u32 *) (tempmem + compfilelen +
    			     sizeof(struct lar_header) + pathlen);
    	     walk++) {
    		csum += ntohl(*walk);
@@ -122,7 +150,7 @@
    	header->checksum = htonl(csum);

/* Write out entry to archive. */
-		entrylen = (filelen + pathlen + sizeof(struct lar_header) +
+		entrylen = (compfilelen + pathlen + sizeof(struct lar_header) +
    		    15) & 0xfffffff0;

fwrite(tempmem, entrylen, 1, archive);
@@ -216,6 +244,7 @@
    	/* construct header */
    	bb=(struct lar_header *)bootblock_header;
    	memcpy(bb->magic, MAGIC, 8);
+		bb->reallen = htonl(bootblock_len);
    	bb->len = htonl(bootblock_len);
    	bb->offset = htonl(bb_header_len);

Index: util/lar/lar.h
===================================================================
--- util/lar/lar.h	(revision 364)
+++ util/lar/lar.h	(working copy)
@@ -48,6 +48,7 @@
  */

#include <stdint.h>
+#include "../../build/config.h"

#define MAGIC "LARCHIVE"
 #define MAX_PATHLEN 1024
@@ -58,6 +59,63 @@
 struct lar_header {
    char magic[8];
    u32 len;
+	u32 reallen;
    u32 checksum;
+	u32 compchecksum;
    u32 offset;
+	/* compression
+	 * 0 = no compression
+	 * 1 = lzma
+	 * 2 = nrv2b
+         */
+	u32 compression;
 };
+
+enum compalgo {none=0, lzma=1, nrv2b=2};
+
+typedef void(*compress_func)(char*,u32,char*,u32*);
+typedef void(*uncompress_func)(char*,char*,u32);
+
+void compress_impossible(char *in, u32 in_len, char *out, u32 *out_len);
+void do_no_compress(char *in, u32 in_len, char *out, u32 *out_len);
+#ifdef CONFIG_COMPRESSION_LZMA
+void do_lzma_compress(char *in, u32 in_len, char* out, u32 *out_len);
+#else
+#define do_lzma_compress compress_impossible
+#endif
+#ifdef CONFIG_COMPRESSION_NRV2B
+void do_nrv2b_compress(char *in, u32 in_len, char *out, u32 *out_len);
+#else
+#define do_nrv2b_compress compress_impossible
+#endif
+
+void uncompress_impossible(char*,char*,u32);
+void do_no_uncompress(char*,char*,u32);
+#ifdef CONFIG_COMPRESSION_LZMA
+void do_lzma_uncompress(char*,char*,u32);
+#else
+#define do_lzma_uncompress uncompress_impossible
+#endif
+#ifdef CONFIG_COMPRESSION_NRV2B
+void do_nrv2b_uncompress(char*,char*,u32);
+#else
+#define do_nrv2b_uncompress uncompress_impossible
+#endif
+
+static compress_func compress_functions[]= {
+        do_no_compress,
+        do_lzma_compress,
+        do_nrv2b_compress
+};
+
+static uncompress_func uncompress_functions[]= {
+        do_no_uncompress,
+        do_lzma_uncompress,
+        do_nrv2b_uncompress
+};
+
+static const char* algo_name[]= {
+ "",
+ "lzma",
+ "nrv2b"
+};
Index: util/lar/lib.c
===================================================================
--- util/lar/lib.c	(revision 364)
+++ util/lar/lib.c	(working copy)
@@ -121,9 +121,15 @@
 {
    struct stat filestat;
    int ret = -1;
+	const char *realname;

+	realname = name;
+	if (strstr(name,"nocompress:") == name) {
+		realname = name+11;
+	}
+
    /* printf("... add_files %s\n", name); */
-	if (stat(name, &filestat) == -1) {
+	if (stat(realname, &filestat) == -1) {
    	fprintf(stderr, "Error getting file attributes of %s\n", name);
    	return -1;
    }
@@ -145,7 +151,7 @@
    }
    // Is it a directory?
    if (S_ISDIR(filestat.st_mode)) {
-		ret = handle_directory(name);
+		ret = handle_directory(realname);
    }
    // Is it a regular file?
    if (S_ISREG(filestat.st_mode)) {
Index: util/lar/extract.c
===================================================================
--- util/lar/extract.c	(revision 364)
+++ util/lar/extract.c	(working copy)
@@ -32,6 +32,15 @@
 #include "lib.h"
 #include "lar.h"

+void uncompress_impossible(char* dst,char* src,u32 len) {
+	fprintf(stderr, "impossible to uncompress data as algorithm isn't compiled in.\n");
+	exit(1);
+}
+
+void do_no_uncompress(char* dst,char* src,u32 len) {
+	memcpy(dst, src, len);
+}
+
 int extract_lar(const char *archivename, struct file *files)
 {
    int archivefile;
@@ -116,8 +125,15 @@
    		exit(1);
    	}

-		fwrite(walk + ntohl(header->offset), ntohl(header->len),
-		       1, file_to_extract);
+		if (ntohl(header->compression) == none) {
+			fwrite(walk + ntohl(header->offset), ntohl(header->len),
+			       1, file_to_extract);
+		} else {
+			char* buf = malloc(ntohl(header->reallen));
+			uncompress_functions[ntohl(header->compression)](buf, walk + ntohl(header->offset), ntohl(header->len));
+			fwrite(buf, ntohl(header->reallen), 1, file_to_extract);
+			free(buf);
+		}
    	fclose(file_to_extract);

walk += (ntohl(header->offset) + ntohl(header->len)
Index: util/lar/list.c
===================================================================
--- util/lar/list.c	(revision 364)
+++ util/lar/list.c	(working copy)
@@ -91,8 +91,20 @@

printf("  %s ", walk + sizeof(struct lar_header));

-		printf("(%d bytes @0x%lx)\n", ntohl(header->len),
-		       (unsigned long)(walk - inmap) + ntohl(header->offset));
+		if (ntohl(header->compression)==none) {
+			printf("(%d bytes @0x%lx)\n",
+			       ntohl(header->len),
+			       (unsigned long)(walk - inmap) +
+			        	ntohl(header->offset));
+		} else {
+			printf("(%d bytes, %s compressed to %d bytes "
+				"@0x%lx)\n",
+			       ntohl(header->reallen),
+			       algo_name[ntohl(header->compression)],
+			       ntohl(header->len),
+			       (unsigned long)(walk - inmap) +
+			        	ntohl(header->offset));
+		}

walk += (ntohl(header->len) + ntohl(header->offset)
    		 - 1) & 0xfffffff0;

[Uwe Hermann]

On Fri, Jun 22, 2007 at 12:13:44AM +0200, Patrick Georgi wrote:
> extend lar format to support compression (incompatible format change!)
> adapt the compression utilities for integration into lar
> add compression capabilities to lar and expose in user interface
> provide a way to mark files as non-compressible
>
> Signed-off-by: Patrick Georgi

Committed in r366 (with minor changes), thanks! I also added your
name as copyright owner in a few files where you made non-trivial
changes.

Uwe.