使用quicklz缩小程序体积
简述
有一个需求是这样的，写的一个程序内置了一个很大的文件（实际就是抓取epsg.io的内容里面的epsg.io.json），这个文件筛选缩减后还有12MB，如果直接内置到程序中，编译后的程序就很大了。
因为这个程序是一个动态库，而使用upx压缩过的动态库有时候会有一些异常问题出现，所以不考虑使用upx进行压缩。
看到了quicklz后，感觉这是个好东西，于是就用这个来进行压缩，把压缩后的数据写入程序中，使用前进行解压即可。使用这个操作之后，程序大小从12MB缩小为不到1.5MB，效果很明显。
压缩和解压代码
关于quicklz的使用，在http://www.quicklz.com/网站上有比较详细的说明，各个编程语言的接口也都有封装好。
更多的可以参考https://github.com/robottwo/quicklz
压缩代码
压缩的代码很简单，因为我这里只做字符串的，所以压缩率还比较高，可以达到12%左右。
压缩的代码如下：
// 压缩字符串src，返回qlz编码格式的内容
std::string quicklz_compress(const std::string& src)
{
qlz_state_compress state;
memset(&state, 0, sizeof(qlz_state_compress));
std::string dst;
char buffer[4096 + 1024];
for(size_t pos = 0;pos<src.size();pos+=4096) {
size_t len = src.size() - pos;
len = len > 4096 ? 4096 : len;
len = qlz_compress(src.data() + pos, buffer, len, &state);
dst.append(buffer,len);
}
return dst;
}
下面是quiz.c里面进行压缩的代码，可供参考
#include "quicklz.h"
#define MAX_BUF_SIZE 1024*1024
#define BUF_BUFFER 400
#define bool int
#define true 1
#define false 0
int stream_compress(FILE *ifile, FILE *ofile)
{
char *file_data, *compressed;
size_t d, c, fd_size, compressed_size;
qlz_state_compress *state_compress = (qlz_state_compress *)malloc(sizeof(qlz_state_compress));
fd_size = MAX_BUF_SIZE;
file_data = (char*) malloc(fd_size);
// allocate MAX_BUF_SIZE + BUF_BUFFER bytes for the destination buffer
compressed_size = MAX_BUF_SIZE + BUF_BUFFER;
compressed = (char*) malloc(compressed_size);
// allocate and initially zero out the states. After this, make sure it is
// preserved across calls and never modified manually
memset(state_compress, 0, sizeof(qlz_state_compress));
// compress the file using MAX_BUF_SIZE packets.
while((d = fread(file_data, 1, MAX_BUF_SIZE, ifile)) != 0)
{
c = qlz_compress(file_data, compressed, d, state_compress);
// the buffer "compressed" now contains c bytes which we could have sent directly to a
// decompressing site for decompression
fwrite(compressed, c, 1, ofile);
}
free(state_compress);
free(compressed);
free(file_data);
return 0;
}
解压代码
解压的速度很快，对程序运行几乎没有影响，比读取文件快多了。
解压代码如下：
std::string quicklz_decompress(const std::string& qlzdata)
{
qlz_state_decompress state;
memset(&state, 0, sizeof(qlz_state_decompress));
std::string dst;
for(size_t pos = 0;ops < qlzdata.size(); ){
// 获取压缩数据段大小
size_t co_size = qlz_size_compressed(qlzdata.data() + pos);
// 获取该压缩段解压后的大小
size_t de_size = qlz_size_decompressed(qlzdata.data() + pos);
std::string buffer(de_size,0);
qlz_decompress(qlzdata.data()+pos, (char*)buffer.data(),&state);
pos += co_size;
dst.append(buffer);
}
return dst;
}
下面是quiz.c里面进行解压的代码，可供参考
int stream_decompress(FILE *ifile, FILE *ofile)
{
char *file_data, *decompressed;
size_t d, c, dc, fd_size, d_size;
qlz_state_decompress *state_decompress = (qlz_state_decompress *)malloc(sizeof(qlz_state_decompress));
// a compressed packet can be at most MAX_BUF_SIZE + BUF_BUFFER bytes if it
// was compressed with this program.
fd_size = MAX_BUF_SIZE + BUF_BUFFER;
file_data = (char*) malloc(fd_size);
// allocate decompression buffer
d_size = fd_size - BUF_BUFFER;
decompressed = (char*) malloc(d_size);
// allocate and initially zero out the scratch buffer. After this, make sure it is
// preserved across calls and never modified manually
memset(state_decompress, 0, sizeof(qlz_state_decompress));
// read 9-byte header to find the size of the entire compressed packet, and
// then read remaining packet
while((c = fread(file_data, 1, 9, ifile)) != 0)
{
// Do we need a bigger decompressed buffer? If the file was compressed
// with segments larger than the default in this program.
dc = qlz_size_decompressed(file_data);
if (dc > (fd_size - BUF_BUFFER)) {
free(file_data);
fd_size = dc + BUF_BUFFER;
file_data = (char*)malloc(fd_size);
}
// Do we need a bigger compressed buffer?
c = qlz_size_compressed(file_data);
if (c > d_size) {
free (decompressed);
d_size = c;
decompressed = (char*)malloc(d_size);
}
fread(file_data + 9, 1, c - 9, ifile);
d = qlz_decompress(file_data, decompressed, state_decompress);
fwrite(decompressed, d, 1, ofile);
}
free(decompressed);
free(state_decompress);
free(file_data);
return 0;
}
将二进制文件生成C数组程序代码
上面的代码用于压缩和解压qlz数据，但是这些数据还需要生成C风格的数组，于是就写了一个小程序来做转换，代码如下：
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
int main(int c,char** v)
{
if(c != 3){
printf("Usage:%s infile outfile\n",v[0]);
return 0;
}
FILE* fin = fopen(v[1],"rb");
if(!fin){
printf("Error:%s Open Failed\n",v[1]);
return 1;
}
FILE* fout = fopen(v[2],"wb");
if(fout){
size_t len = 0;
unsigned char buffer[16];
char strbuffer[1024] = "const unsigned char carr_xxx[] = {";
fwrite(strbuffer,1,strlen(strbuffer),fout);
while((len = fread(buffer,1 ,sizeof buffer,fin)) != 0){
strbuffer[0] = '\n';
strbuffer[1] = '\t';
for(size_t i = 0, offset = 2; i < len; ++i) {
offset += sprintf(&strbuffer[offset],"%hhu,",buffer[i]);
}
fwrite(strbuffer,1,strlen(strbuffer),fout);
}
if(strbuffer[0] != 'c'){
fseek(fout,-1, SEEK_CUR);
}
strcpy(strbuffer,"\n};\n");
fwrite(strbuffer,1,strlen(strbuffer),fout);
fclose(fout);
}
fclose(fin);
return 0;
}
