Hi,
I attach a patch to enable compression and decompression in lar. After some discussion with Uwe, this patch cleans up some style issues (I hope I got them all, please bear with me if not!), presents some more information to the user and in a cleaner way, and also provides a way to mark files as "not compress", by prefixing their filename with "nocompress:"
Comments appreciated.
Regards, Patrick Georgi
- extend lar format to support compression (incompatible format change!) - adapt the compression utilities for integration into lar - add compression capabilities to lar and expose in user interface - provide a way to mark files as non-compressible
Signed-off-by: Patrick Georgi patrick@georgi-clan.de
Index: include/lar.h =================================================================== --- include/lar.h (revision 364) +++ include/lar.h (working copy) @@ -58,8 +58,11 @@ struct lar_header { char magic[8]; u32 len; + u32 reallen; u32 checksum; + u32 compchecksum; u32 offset; + u32 compression; };
struct mem_file { Index: util/lzma/minilzma.cc =================================================================== --- util/lzma/minilzma.cc (revision 364) +++ util/lzma/minilzma.cc (working copy) @@ -281,13 +281,19 @@ #else extern "C" {
-void do_lzma_compress(char* in, unsigned long in_len, char* out, unsigned long* out_len) { +void do_lzma_compress(char *in, unsigned long in_len, char *out, unsigned long *out_len) { std::vector<unsigned char> result; result = LZMACompress(std::vector<unsigned char>(in,in+in_len)); *out_len = result.size(); std::memcpy(out, &result[0], *out_len); }
+void do_lzma_uncompress(char *dst, char *src, unsigned long len) { + std::vector<unsigned char> result; + result = LZMADeCompress(std::vector<unsigned char>(src,src+len)); + std::memcpy(dst, &result[0], result.size()); } + +} #endif
Index: util/nrv2b/nrv2b.c =================================================================== --- util/nrv2b/nrv2b.c (revision 364) +++ util/nrv2b/nrv2b.c (working copy) @@ -1304,6 +1304,60 @@ out = malloc(*out_len); ucl_nrv2b_99_compress(in, in_len, out, out_len, 0 ); } + +void do_nrv2b_uncompress(char* dst, char* src, unsigned long len) { + unsigned long ilen = 0, olen = 0, last_m_off = 1; + for (;;) { + unsigned int m_off, m_len; + while(GETBIT(bb, src, ilen)) { + FAIL(ilen >= src_len, "input overrun"); + FAIL(olen >= dst_len, "output overrun"); + dst[olen++] = src[ilen++]; + } + m_off = 1; + do { + m_off = m_off*2 + GETBIT(bb, src, ilen); + FAIL(ilen >= src_len, "input overrun"); + FAIL(m_off > 0xffffffU +3, "lookbehind overrun"); + } while (!GETBIT(bb, src, ilen)); + if (m_off == 2) + { + m_off = last_m_off; + } + else + { + FAIL(ilen >= src_len, "input overrun"); + m_off = (m_off - 3)*256 + src[ilen++]; + if(m_off == 0xffffffffU) + break; + last_m_off = ++m_off; + } + m_len = GETBIT(bb, src, ilen); + m_len = m_len*2 + GETBIT(bb, src, ilen); + if (m_len == 0) + { + m_len++; + do { + m_len = m_len*2 + GETBIT(bb, src, ilen); + FAIL(ilen >= src_len, "input overrun"); + FAIL(m_len >= dst_len, "output overrun"); + } while(!GETBIT(bb, src, ilen)); + m_len += 2; + } + m_len += (m_off > 0xd00); + FAIL(olen + m_len > dst_len, "output overrun"); + FAIL(m_off > olen, "lookbeind overrun"); + { + const uint8_t *m_pos; + m_pos = dst + olen - m_off; + dst[olen++] = *m_pos++; + do { + dst[olen++] = *m_pos++; + } while(--m_len > 0); + } + } + FAIL(ilen < src_len, "input not consumed"); +} #endif
#ifdef DECODE Index: util/lar/lar.c =================================================================== --- util/lar/lar.c (revision 364) +++ util/lar/lar.c (working copy) @@ -39,6 +39,7 @@ static int isverbose = 0; static long larsize = 0; static char *bootblock = NULL; +enum compalgo algo = none;
static void usage(char *name) { @@ -72,6 +73,7 @@
static struct option long_options[] = { {"create", 0, 0, 'c'}, + {"compress-algo", 1, 0, 'C'}, {"extract", 0, 0, 'x'}, {"list", 0, 0, 'l'}, {"size", 1, 0, 's'}, @@ -87,12 +89,20 @@ exit(1); }
- while ((opt = getopt_long(argc, argv, "cxls:b:vVh?", + while ((opt = getopt_long(argc, argv, "cC:xls:b:vVh?", long_options, &option_index)) != EOF) { switch (opt) { case 'c': larmode = CREATE; break; + case 'C': + if (strcmp("lzma",optarg)==0) { + algo=lzma; + } + if (strcmp("nrv2b",optarg)==0) { + algo=nrv2b; + } + break; case 'l': larmode = LIST; break; Index: util/lar/create.c =================================================================== --- util/lar/create.c (revision 364) +++ util/lar/create.c (working copy) @@ -34,6 +34,17 @@ #include "lib.h" #include "lar.h"
+extern enum compalgo algo; +void compress_impossible(char* in, u32 in_len, char* out, u32* out_len) { + fprintf(stderr,"the selected compression algorithm wasn't compiled in.\n"); + exit(1); +} +void do_no_compress(char* in, u32 in_len, char* out, u32* out_len) { + memcpy(out,in,in_len); + out_len[0]=in_len; +} + + int create_lar(const char *archivename, struct file *files) { int i, ret; @@ -41,14 +52,16 @@ int bb_header_len = 0; FILE *archive, *source; char *tempmem; - char *filebuf; + char *filebuf, *filetarget; char *pathname; u32 *walk; u32 csum; int pathlen, entrylen, filelen; + u32 compfilelen; long currentsize = 0; struct lar_header *header; struct stat statbuf; + enum compalgo thisalgo;
if (!files) { fprintf(stderr, "No files for archive %s\n", archivename); @@ -68,6 +81,12 @@ while (files) { char *name = files->name;
+ thisalgo = algo; + + if (strstr(name,"nocompress:") == name) { + name += 11; + thisalgo = none; + } /* skip ./ if available */ if (name[0] == '.' && name[1] == '/') name += 2; @@ -97,24 +116,33 @@ pathlen = (pathlen + 15) & 0xfffffff0;/* Align to 16 bytes. */
/* Read file into memory. */ - filebuf = pathname + pathlen; + filebuf = malloc(filelen); + filetarget = pathname + pathlen; source = fopen(name, "r"); if (!source) { fprintf(stderr, "No such file %s\n", name); exit(1); } - fread(filebuf, statbuf.st_size, 1, source); + fread(filebuf, filelen, 1, source); fclose(source); + compress_functions[thisalgo](filebuf, filelen, filetarget, &compfilelen); + if ((compfilelen >= filelen) && (thisalgo != none)) { + thisalgo = none; + compress_functions[thisalgo](filebuf, filelen, filetarget, &compfilelen); + } + free(filebuf);
/* Create correct header. */ memcpy(header, MAGIC, 8); - header->len = htonl(statbuf.st_size); + header->compression = htonl(thisalgo); + header->reallen = htonl(filelen); + header->len = htonl(compfilelen); header->offset = htonl(sizeof(struct lar_header) + pathlen);
/* Calculate checksum. */ csum = 0; for (walk = (u32 *) tempmem; - walk < (u32 *) (tempmem + statbuf.st_size + + walk < (u32 *) (tempmem + compfilelen + sizeof(struct lar_header) + pathlen); walk++) { csum += ntohl(*walk); @@ -122,7 +150,7 @@ header->checksum = htonl(csum);
/* Write out entry to archive. */ - entrylen = (filelen + pathlen + sizeof(struct lar_header) + + entrylen = (compfilelen + pathlen + sizeof(struct lar_header) + 15) & 0xfffffff0;
fwrite(tempmem, entrylen, 1, archive); @@ -216,6 +244,7 @@ /* construct header */ bb=(struct lar_header *)bootblock_header; memcpy(bb->magic, MAGIC, 8); + bb->reallen = htonl(bootblock_len); bb->len = htonl(bootblock_len); bb->offset = htonl(bb_header_len);
Index: util/lar/lar.h =================================================================== --- util/lar/lar.h (revision 364) +++ util/lar/lar.h (working copy) @@ -48,6 +48,7 @@ */
#include <stdint.h> +#include "../../build/config.h"
#define MAGIC "LARCHIVE" #define MAX_PATHLEN 1024 @@ -58,6 +59,63 @@ struct lar_header { char magic[8]; u32 len; + u32 reallen; u32 checksum; + u32 compchecksum; u32 offset; + /* compression + * 0 = no compression + * 1 = lzma + * 2 = nrv2b + */ + u32 compression; }; + +enum compalgo {none=0, lzma=1, nrv2b=2}; + +typedef void(*compress_func)(char*,u32,char*,u32*); +typedef void(*uncompress_func)(char*,char*,u32); + +void compress_impossible(char *in, u32 in_len, char *out, u32 *out_len); +void do_no_compress(char *in, u32 in_len, char *out, u32 *out_len); +#ifdef CONFIG_COMPRESSION_LZMA +void do_lzma_compress(char *in, u32 in_len, char* out, u32 *out_len); +#else +#define do_lzma_compress compress_impossible +#endif +#ifdef CONFIG_COMPRESSION_NRV2B +void do_nrv2b_compress(char *in, u32 in_len, char *out, u32 *out_len); +#else +#define do_nrv2b_compress compress_impossible +#endif + +void uncompress_impossible(char*,char*,u32); +void do_no_uncompress(char*,char*,u32); +#ifdef CONFIG_COMPRESSION_LZMA +void do_lzma_uncompress(char*,char*,u32); +#else +#define do_lzma_uncompress uncompress_impossible +#endif +#ifdef CONFIG_COMPRESSION_NRV2B +void do_nrv2b_uncompress(char*,char*,u32); +#else +#define do_nrv2b_uncompress uncompress_impossible +#endif + +static compress_func compress_functions[]= { + do_no_compress, + do_lzma_compress, + do_nrv2b_compress +}; + +static uncompress_func uncompress_functions[]= { + do_no_uncompress, + do_lzma_uncompress, + do_nrv2b_uncompress +}; + +static const char* algo_name[]= { + "", + "lzma", + "nrv2b" +}; Index: util/lar/lib.c =================================================================== --- util/lar/lib.c (revision 364) +++ util/lar/lib.c (working copy) @@ -121,9 +121,15 @@ { struct stat filestat; int ret = -1; + const char *realname;
+ realname = name; + if (strstr(name,"nocompress:") == name) { + realname = name+11; + } + /* printf("... add_files %s\n", name); */ - if (stat(name, &filestat) == -1) { + if (stat(realname, &filestat) == -1) { fprintf(stderr, "Error getting file attributes of %s\n", name); return -1; } @@ -145,7 +151,7 @@ } // Is it a directory? if (S_ISDIR(filestat.st_mode)) { - ret = handle_directory(name); + ret = handle_directory(realname); } // Is it a regular file? if (S_ISREG(filestat.st_mode)) { Index: util/lar/extract.c =================================================================== --- util/lar/extract.c (revision 364) +++ util/lar/extract.c (working copy) @@ -32,6 +32,15 @@ #include "lib.h" #include "lar.h"
+void uncompress_impossible(char* dst,char* src,u32 len) { + fprintf(stderr, "impossible to uncompress data as algorithm isn't compiled in.\n"); + exit(1); +} + +void do_no_uncompress(char* dst,char* src,u32 len) { + memcpy(dst, src, len); +} + int extract_lar(const char *archivename, struct file *files) { int archivefile; @@ -116,8 +125,15 @@ exit(1); }
- fwrite(walk + ntohl(header->offset), ntohl(header->len), - 1, file_to_extract); + if (ntohl(header->compression) == none) { + fwrite(walk + ntohl(header->offset), ntohl(header->len), + 1, file_to_extract); + } else { + char* buf = malloc(ntohl(header->reallen)); + uncompress_functions[ntohl(header->compression)](buf, walk + ntohl(header->offset), ntohl(header->len)); + fwrite(buf, ntohl(header->reallen), 1, file_to_extract); + free(buf); + } fclose(file_to_extract);
walk += (ntohl(header->offset) + ntohl(header->len) Index: util/lar/list.c =================================================================== --- util/lar/list.c (revision 364) +++ util/lar/list.c (working copy) @@ -91,8 +91,20 @@
printf(" %s ", walk + sizeof(struct lar_header));
- printf("(%d bytes @0x%lx)\n", ntohl(header->len), - (unsigned long)(walk - inmap) + ntohl(header->offset)); + if (ntohl(header->compression)==none) { + printf("(%d bytes @0x%lx)\n", + ntohl(header->len), + (unsigned long)(walk - inmap) + + ntohl(header->offset)); + } else { + printf("(%d bytes, %s compressed to %d bytes " + "@0x%lx)\n", + ntohl(header->reallen), + algo_name[ntohl(header->compression)], + ntohl(header->len), + (unsigned long)(walk - inmap) + + ntohl(header->offset)); + }
walk += (ntohl(header->len) + ntohl(header->offset) - 1) & 0xfffffff0;