micropython/extmod/moddeflate.c

416 lines
14 KiB
C

/*
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2023 Jim Mussared
*
* Based on extmod/modzlib.c
* Copyright (c) 2014-2016 Paul Sokolovsky
* Copyright (c) 2021-2023 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <stdio.h>
#include <string.h>
#include "py/runtime.h"
#include "py/stream.h"
#include "py/mperrno.h"
#if MICROPY_PY_DEFLATE
#include "lib/uzlib/uzlib.h"
#if 0 // print debugging info
#define DEBUG_printf DEBUG_printf
#else // don't print debugging info
#define DEBUG_printf(...) (void)0
#endif
typedef enum {
DEFLATEIO_FORMAT_MIN = 0,
DEFLATEIO_FORMAT_AUTO = DEFLATEIO_FORMAT_MIN, // Read mode this means auto-detect zlib/gzip, write mode this means RAW.
DEFLATEIO_FORMAT_RAW = 1,
DEFLATEIO_FORMAT_ZLIB = 2,
DEFLATEIO_FORMAT_GZIP = 3,
DEFLATEIO_FORMAT_MAX = DEFLATEIO_FORMAT_GZIP,
} deflateio_format_t;
// This is used when the wbits is unset in the DeflateIO constructor. Default
// to the smallest window size (faster compression, less RAM usage, etc).
const int DEFLATEIO_DEFAULT_WBITS = 8;
typedef struct {
void *window;
uzlib_uncomp_t decomp;
bool eof;
} mp_obj_deflateio_read_t;
#if MICROPY_PY_DEFLATE_COMPRESS
typedef struct {
void *window;
size_t input_len;
uint32_t input_checksum;
uzlib_lz77_state_t lz77;
} mp_obj_deflateio_write_t;
#endif
typedef struct {
mp_obj_base_t base;
mp_obj_t stream;
uint8_t format : 2;
uint8_t window_bits : 4;
bool close : 1;
mp_obj_deflateio_read_t *read;
#if MICROPY_PY_DEFLATE_COMPRESS
mp_obj_deflateio_write_t *write;
#endif
} mp_obj_deflateio_t;
STATIC int deflateio_read_stream(void *data) {
mp_obj_deflateio_t *self = data;
const mp_stream_p_t *stream = mp_get_stream(self->stream);
int err;
byte c;
mp_uint_t out_sz = stream->read(self->stream, &c, 1, &err);
if (out_sz == MP_STREAM_ERROR) {
mp_raise_OSError(err);
}
if (out_sz == 0) {
mp_raise_type(&mp_type_EOFError);
}
return c;
}
STATIC bool deflateio_init_read(mp_obj_deflateio_t *self) {
if (self->read) {
return true;
}
mp_get_stream_raise(self->stream, MP_STREAM_OP_READ);
self->read = m_new_obj(mp_obj_deflateio_read_t);
memset(&self->read->decomp, 0, sizeof(self->read->decomp));
self->read->decomp.source_read_data = self;
self->read->decomp.source_read_cb = deflateio_read_stream;
self->read->eof = false;
// Don't modify self->window_bits as it may also be used for write.
int wbits = self->window_bits;
if (self->format == DEFLATEIO_FORMAT_RAW) {
if (wbits == 0) {
// The docs recommends always setting wbits explicitly when using
// RAW, but we still allow a default.
wbits = DEFLATEIO_DEFAULT_WBITS;
}
} else {
// Parse the header if we're in NONE/ZLIB/GZIP modes.
int header_wbits;
int header_type = uzlib_parse_zlib_gzip_header(&self->read->decomp, &header_wbits);
if (header_type < 0) {
// Stream header was invalid.
return false;
}
if ((self->format == DEFLATEIO_FORMAT_ZLIB && header_type != UZLIB_HEADER_ZLIB) || (self->format == DEFLATEIO_FORMAT_GZIP && header_type != UZLIB_HEADER_GZIP)) {
// Not what we expected.
return false;
}
// header_wbits will either be 15 (gzip) or 8-15 (zlib).
if (wbits == 0 || header_wbits < wbits) {
// If the header specified something lower, then use that instead.
// No point doing a bigger allocation than we need to.
wbits = header_wbits;
}
}
size_t window_len = 1 << wbits;
self->read->window = m_new(uint8_t, window_len);
uzlib_uncompress_init(&self->read->decomp, self->read->window, window_len);
return true;
}
#if MICROPY_PY_DEFLATE_COMPRESS
STATIC void deflateio_out_byte(void *data, uint8_t b) {
mp_obj_deflateio_t *self = data;
const mp_stream_p_t *stream = mp_get_stream(self->stream);
int err;
mp_uint_t ret = stream->write(self->stream, &b, 1, &err);
if (ret == MP_STREAM_ERROR) {
mp_raise_OSError(err);
}
}
STATIC bool deflateio_init_write(mp_obj_deflateio_t *self) {
if (self->write) {
return true;
}
const mp_stream_p_t *stream = mp_get_stream_raise(self->stream, MP_STREAM_OP_WRITE);
self->write = m_new_obj(mp_obj_deflateio_write_t);
self->write->input_len = 0;
int wbits = self->window_bits;
if (wbits == 0) {
// Same default wbits for all formats.
wbits = DEFLATEIO_DEFAULT_WBITS;
}
size_t window_len = 1 << wbits;
self->write->window = m_new(uint8_t, window_len);
uzlib_lz77_init(&self->write->lz77, self->write->window, window_len);
self->write->lz77.dest_write_data = self;
self->write->lz77.dest_write_cb = deflateio_out_byte;
// Write header if needed.
mp_uint_t ret = 0;
int err;
if (self->format == DEFLATEIO_FORMAT_ZLIB) {
// -----CMF------ ----------FLG---------------
// CINFO(5) CM(3) FLEVEL(2) FDICT(1) FCHECK(5)
uint8_t buf[] = { 0x08, 0x80 }; // CM=2 (deflate), FLEVEL=2 (default), FDICT=0 (no dictionary)
buf[0] |= MAX(wbits - 8, 1) << 4; // base-2 logarithm of the LZ77 window size, minus eight.
buf[1] |= 31 - ((buf[0] * 256 + buf[1]) % 31); // (CMF*256 + FLG) % 31 == 0.
ret = stream->write(self->stream, buf, sizeof(buf), &err);
self->write->input_checksum = 1; // ADLER32
} else if (self->format == DEFLATEIO_FORMAT_GZIP) {
// ID1(8) ID2(8) CM(8) ---FLG--- MTIME(32) XFL(8) OS(8)
// FLG: x x x FCOMMENT FNAME FEXTRA FHCRC FTEXT
uint8_t buf[] = { 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03 }; // MTIME=0, XFL=4 (fastest), OS=3 (unix)
ret = stream->write(self->stream, buf, sizeof(buf), &err);
self->write->input_checksum = ~0; // CRC32
}
if (ret == MP_STREAM_ERROR) {
return false;
}
// Write starting block.
uzlib_start_block(&self->write->lz77);
return true;
}
#endif
STATIC mp_obj_t deflateio_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args_in) {
// args: stream, format=NONE, wbits=0, close=False
mp_arg_check_num(n_args, n_kw, 1, 4, false);
mp_int_t format = n_args > 1 ? mp_obj_get_int(args_in[1]) : DEFLATEIO_FORMAT_AUTO;
mp_int_t wbits = n_args > 2 ? mp_obj_get_int(args_in[2]) : 0;
if (format < DEFLATEIO_FORMAT_MIN || format > DEFLATEIO_FORMAT_MAX) {
mp_raise_ValueError(MP_ERROR_TEXT("format"));
}
if (wbits != 0 && (wbits < 5 || wbits > 15)) {
mp_raise_ValueError(MP_ERROR_TEXT("wbits"));
}
mp_obj_deflateio_t *self = mp_obj_malloc(mp_obj_deflateio_t, type);
self->stream = args_in[0];
self->format = format;
self->window_bits = wbits;
self->read = NULL;
#if MICROPY_PY_DEFLATE_COMPRESS
self->write = NULL;
#endif
self->close = n_args > 3 ? mp_obj_is_true(args_in[3]) : false;
return MP_OBJ_FROM_PTR(self);
}
STATIC mp_uint_t deflateio_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) {
mp_obj_deflateio_t *self = MP_OBJ_TO_PTR(o_in);
if (self->stream == MP_OBJ_NULL || !deflateio_init_read(self)) {
*errcode = MP_EINVAL;
return MP_STREAM_ERROR;
}
if (self->read->eof) {
return 0;
}
self->read->decomp.dest = buf;
self->read->decomp.dest_limit = (uint8_t *)buf + size;
int st = uzlib_uncompress_chksum(&self->read->decomp);
if (st == UZLIB_DONE) {
self->read->eof = true;
}
if (st < 0) {
DEBUG_printf("uncompress error=" INT_FMT "\n", st);
*errcode = MP_EINVAL;
return MP_STREAM_ERROR;
}
return self->read->decomp.dest - (uint8_t *)buf;
}
#if MICROPY_PY_DEFLATE_COMPRESS
STATIC mp_uint_t deflateio_write(mp_obj_t self_in, const void *buf, mp_uint_t size, int *errcode) {
mp_obj_deflateio_t *self = MP_OBJ_TO_PTR(self_in);
if (self->stream == MP_OBJ_NULL || !deflateio_init_write(self)) {
*errcode = MP_EINVAL;
return MP_STREAM_ERROR;
}
self->write->input_len += size;
if (self->format == DEFLATEIO_FORMAT_ZLIB) {
self->write->input_checksum = uzlib_adler32(buf, size, self->write->input_checksum);
} else if (self->format == DEFLATEIO_FORMAT_GZIP) {
self->write->input_checksum = uzlib_crc32(buf, size, self->write->input_checksum);
}
uzlib_lz77_compress(&self->write->lz77, buf, size);
return size;
}
static inline void put_le32(char *buf, uint32_t value) {
buf[0] = value & 0xff;
buf[1] = value >> 8 & 0xff;
buf[2] = value >> 16 & 0xff;
buf[3] = value >> 24 & 0xff;
}
static inline void put_be32(char *buf, uint32_t value) {
buf[3] = value & 0xff;
buf[2] = value >> 8 & 0xff;
buf[1] = value >> 16 & 0xff;
buf[0] = value >> 24 & 0xff;
}
#endif
STATIC mp_uint_t deflateio_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_t arg, int *errcode) {
if (request == MP_STREAM_CLOSE) {
mp_obj_deflateio_t *self = MP_OBJ_TO_PTR(self_in);
mp_uint_t ret = 0;
if (self->stream != MP_OBJ_NULL) {
#if MICROPY_PY_DEFLATE_COMPRESS
if (self->write) {
uzlib_finish_block(&self->write->lz77);
const mp_stream_p_t *stream = mp_get_stream(self->stream);
// Write footer if needed.
if (self->format == DEFLATEIO_FORMAT_ZLIB || self->format == DEFLATEIO_FORMAT_GZIP) {
char footer[8];
size_t footer_len;
if (self->format == DEFLATEIO_FORMAT_ZLIB) {
put_be32(&footer[0], self->write->input_checksum);
footer_len = 4;
} else { // DEFLATEIO_FORMAT_GZIP
put_le32(&footer[0], ~self->write->input_checksum);
put_le32(&footer[4], self->write->input_len);
footer_len = 8;
}
if (stream->write(self->stream, footer, footer_len, errcode) == MP_STREAM_ERROR) {
ret = MP_STREAM_ERROR;
}
}
}
#endif
// Only close the stream if required. e.g. when using io.BytesIO
// it needs to stay open so that getvalue() can be called.
if (self->close) {
mp_stream_close(self->stream);
}
// Either way, free the reference to the stream.
self->stream = MP_OBJ_NULL;
}
return ret;
} else {
*errcode = MP_EINVAL;
return MP_STREAM_ERROR;
}
}
STATIC const mp_stream_p_t deflateio_stream_p = {
.read = deflateio_read,
#if MICROPY_PY_DEFLATE_COMPRESS
.write = deflateio_write,
#endif
.ioctl = deflateio_ioctl,
};
#if !MICROPY_ENABLE_DYNRUNTIME
STATIC const mp_rom_map_elem_t deflateio_locals_dict_table[] = {
{ MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) },
{ MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) },
{ MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) },
#if MICROPY_PY_DEFLATE_COMPRESS
{ MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) },
#endif
{ MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) },
{ MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) },
{ MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) },
};
STATIC MP_DEFINE_CONST_DICT(deflateio_locals_dict, deflateio_locals_dict_table);
STATIC MP_DEFINE_CONST_OBJ_TYPE(
deflateio_type,
MP_QSTR_DeflateIO,
MP_TYPE_FLAG_NONE,
make_new, deflateio_make_new,
protocol, &deflateio_stream_p,
locals_dict, &deflateio_locals_dict
);
STATIC const mp_rom_map_elem_t mp_module_deflate_globals_table[] = {
{ MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_deflate) },
{ MP_ROM_QSTR(MP_QSTR_DeflateIO), MP_ROM_PTR(&deflateio_type) },
{ MP_ROM_QSTR(MP_QSTR_AUTO), MP_ROM_INT(DEFLATEIO_FORMAT_AUTO) },
{ MP_ROM_QSTR(MP_QSTR_RAW), MP_ROM_INT(DEFLATEIO_FORMAT_RAW) },
{ MP_ROM_QSTR(MP_QSTR_ZLIB), MP_ROM_INT(DEFLATEIO_FORMAT_ZLIB) },
{ MP_ROM_QSTR(MP_QSTR_GZIP), MP_ROM_INT(DEFLATEIO_FORMAT_GZIP) },
};
STATIC MP_DEFINE_CONST_DICT(mp_module_deflate_globals, mp_module_deflate_globals_table);
const mp_obj_module_t mp_module_deflate = {
.base = { &mp_type_module },
.globals = (mp_obj_dict_t *)&mp_module_deflate_globals,
};
MP_REGISTER_MODULE(MP_QSTR_deflate, mp_module_deflate);
#endif // !MICROPY_ENABLE_DYNRUNTIME
// Source files #include'd here to make sure they're compiled in
// only if the module is enabled.
#include "lib/uzlib/tinflate.c"
#include "lib/uzlib/header.c"
#include "lib/uzlib/adler32.c"
#include "lib/uzlib/crc32.c"
#if MICROPY_PY_DEFLATE_COMPRESS
#include "lib/uzlib/lz77.c"
#endif
#endif // MICROPY_PY_DEFLATE