Release: v2.1.6 - Added VaptVupt 2.40

This commit is contained in:
Cristian Cezar Moisés 2026-04-22 03:46:59 -03:00
commit d4660e6539
41 changed files with 4309 additions and 741 deletions

View file

@ -1,25 +1,18 @@
/* VaptVupt codec — originally Apache-2.0 by Cristian Cezar Moisés
* Integrated into Zupt MIT License
* Copyright (c) 2026 Cristian Cezar Moisés
* SPDX-License-Identifier: MIT AND Apache-2.0
*/
/*
* VaptVupt Codec Next-generation lossless compression
* Public API and data structures
*
* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright 2026 Cristian.
* Zero dependencies. Pure C11.
*/
#ifndef VAPTVUPT_H
#define VAPTVUPT_H
#include <stdint.h>
#include "vv_platform.h"
#include <stddef.h>
/* VAPTVUPT: When integrated into Zupt, pull in zupt_xxh64 declaration */
#ifndef VV_STANDALONE
#include "zupt.h"
#endif
#ifdef __cplusplus
extern "C" {
#endif
@ -81,6 +74,12 @@ typedef enum {
#define VV_ENTROPY_ANS4 0x49 /* 'I' — tANS 4-way interleaved (v0.6+) */
#define VV_ENTROPY_CTX 0x43 /* 'C' — tANS order-1 context model (v0.7+) */
#define VV_ENTROPY_SEQ 0x53 /* 'S' — sequence coding: ANS on lits+ml+of (v0.8+) */
#define VV_ENTROPY_SEQ_V2 0x54 /* 'T' — same as 'S' but with min_match=3
* for binary-data compression parity with
* gzip-9. Shifts ml_base[] down by 1 across
* all 36 codes; every other field unchanged.
* Added in v2.33.0 (decode); encoder in a
* future release. */
/* Block header accessors (2-bit type, 1-bit last, 21-bit size) */
static inline vv_block_type_t vv_bh_type(uint32_t h) { return (vv_block_type_t)(h & 3); }
@ -187,6 +186,9 @@ typedef struct {
uint8_t window_log; /* 0 = auto (20 for balanced, 24 for extreme) */
int checksum; /* 1 = compute XXH64 */
int verbose;
int format_v2; /* 1 = produce 'T' tag blocks (min_match=3) for
* better real-binary ratio. Requires decoder
* v2.33.0+. Default 0 for back-compat. */
} vv_options_t;
static inline void vv_default_options(vv_options_t *o) {
@ -194,10 +196,11 @@ static inline void vv_default_options(vv_options_t *o) {
o->window_log = 0;
o->checksum = 1;
o->verbose = 0;
o->format_v2 = 0;
}
/* ═══════════════════════════════════════════════════════════════
* PUBLIC API
* PUBLIC API ONE-SHOT
* */
/* Compress src[0..src_len-1] into dst[0..dst_cap-1].
@ -211,36 +214,238 @@ int64_t vv_compress(const uint8_t *src, size_t src_len,
int64_t vv_decompress(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap);
/* Flags for vv_decompress_flags (bitmask) */
#define VV_DECOMPRESS_DEFAULT 0x0
#define VV_DECOMPRESS_SKIP_CHECKSUM 0x1 /* Skip XXH64 footer verification.
*
* Use when the caller has its own
* integrity protection (e.g. AES-GCM
* wrapping the compressed data, as in
* Zupt backups). On RAW/random-data
* inputs where XXH64 dominates decode
* time, this flag delivers a ~2× speedup.
*
* SAFETY: only set when another layer
* already detects tampering/corruption.
* Without any integrity check, silent
* data corruption can go undetected. */
/* Decompress with flags. Returns decompressed size, or negative error code.
* Equivalent to vv_decompress() when flags == VV_DECOMPRESS_DEFAULT. */
int64_t vv_decompress_flags(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap,
uint32_t flags);
/* Compute upper bound on compressed size for src_len input bytes. */
size_t vv_compress_bound(size_t src_len);
/* ═══════════════════════════════════════════════════════════════
* MULTI-THREADED COMPRESSION
*
* Compresses large inputs in parallel by splitting into independent
* frames (each a valid .vv frame on its own concatenated output
* is a valid .vv file that vv_decompress handles natively as a
* multi-frame stream).
*
* Requires the library to be built with VV_ENABLE_THREADS (and
* linked with -lpthread on POSIX). If threads are not available,
* the function falls back to sequential single-threaded encoding,
* producing bit-identical output to vv_compress.
*
* Tradeoff: multi-frame output is ~0.5-2% larger than a single
* vv_compress frame because cross-frame match history is lost. Use
* for inputs 4 MB where parallel speedup outweighs the ratio cost.
* */
/* Compress src in parallel using up to nthreads worker threads.
* If nthreads is 0, uses the number of online CPUs (or 1 if that
* cannot be determined). If the library was built without threading,
* this acts exactly like vv_compress (nthreads is ignored).
*
* chunk_size controls the frame split size must be 1 MB for
* reasonable compression ratio. If 0, defaults to 4 MB.
*
* Returns compressed size on success, negative error code on failure. */
int64_t vv_compress_mt(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap,
const vv_options_t *opts,
unsigned int nthreads,
size_t chunk_size);
/* Frame info extracted from the first 16 bytes of a compressed stream.
* Populated by vv_get_frame_info(). */
typedef struct {
uint8_t version; /* Format version */
uint8_t has_checksum; /* Non-zero if frame has XXH64 footer */
uint8_t mode_hint; /* Compression mode used (informational) */
uint8_t window_log; /* Window size = 1 << window_log */
uint64_t content_size; /* Uncompressed size if known (0 = unknown) */
} vv_frame_info_t;
/* Parse the first 16 bytes of a compressed stream to extract frame
* metadata. Requires src_len >= 16. Useful for pre-allocating the
* output buffer when content_size is known (e.g., streams produced
* by the one-shot vv_compress API always carry content_size).
*
* Returns VV_OK on success, negative error code on bad magic /
* unsupported version / too-short input. */
int vv_get_frame_info(const uint8_t *src, size_t src_len,
vv_frame_info_t *info);
/* ═══════════════════════════════════════════════════════════════
* STREAMING API for large files, memory-constrained use, or
* when the full input/output isn't known in advance.
*
* Compress:
* ctx = vv_cstream_create(&opts);
* for each chunk: vv_cstream_compress_chunk(ctx, chunk, len, dst, dst_cap, &written, is_last);
* vv_cstream_destroy(ctx);
*
* Decompress:
* ctx = vv_dstream_create();
* for each incoming block: vv_dstream_decompress_chunk(ctx, src, len, dst, dst_cap, &read, &written);
* vv_dstream_destroy(ctx);
*
* Compression is block-at-a-time: caller accumulates source data in
* chunks of up to VV_MAX_BLOCK_SIZE (1 MB). Each call to
* vv_cstream_compress_chunk emits one compressed block (or the frame
* header on the first call, and the frame footer on the last).
*
* Decompression accepts arbitrary byte chunks and emits decoded bytes
* as blocks complete. Partial blocks are buffered internally.
* */
/* Opaque stream context types */
typedef struct vv_cstream_s vv_cstream_t;
typedef struct vv_dstream_s vv_dstream_t;
/* Create a new compression stream context.
* Returns NULL on allocation failure.
* If opts is NULL, uses default options (balanced mode, checksum=1).
* The context holds the matcher state; cross-block rep-match history
* and hash tables are preserved across chunks for optimal ratio. */
vv_cstream_t *vv_cstream_create(const vv_options_t *opts);
/* Reset a compression stream for reuse. Clears the matcher state,
* rep-match offsets, checksum accumulator, and emission flag so the
* context can be used to compress a new independent frame.
* Scratch buffers are preserved this is the fast path for
* per-file compression (e.g., backup tools compressing many small
* files), avoiding per-file allocation cost.
*
* If opts is NULL, reuses the options from the last create/reset.
* If opts is non-NULL, applies new options but window_log cannot
* change (would require re-allocating matcher tables). */
int vv_cstream_reset(vv_cstream_t *ctx, const vv_options_t *opts);
/* Compress one chunk of source into dst. chunk_len must be ≤
* VV_MAX_BLOCK_SIZE (1 MB). Set is_last=1 on the final call to emit
* the frame footer (checksum if enabled).
*
* Writes at most dst_cap bytes to dst; sets *written to the actual
* number of bytes emitted. Caller must ensure dst_cap
* vv_compress_bound(chunk_len) + 24 (frame header + footer).
*
* On the first call, the frame header is emitted before the first
* block. On the last call, the frame footer (if checksum enabled) is
* emitted after the final block.
*
* Returns VV_OK (0) on success, negative error code on failure. */
int vv_cstream_compress_chunk(vv_cstream_t *ctx,
const uint8_t *chunk, size_t chunk_len,
uint8_t *dst, size_t dst_cap,
size_t *written, int is_last);
/* Destroy a compression stream context and free all resources. */
void vv_cstream_destroy(vv_cstream_t *ctx);
/* Create a new decompression stream context.
* Returns NULL on allocation failure. */
vv_dstream_t *vv_dstream_create(void);
/* Reset a decompression stream for reuse. Clears state so the same
* context can decompress another independent frame. Internal buffer
* is preserved (but emptied), avoiding per-frame allocation cost. */
int vv_dstream_reset(vv_dstream_t *ctx);
/* Decompress a chunk of input. src may contain partial or multiple
* blocks; internal buffer holds incomplete blocks until enough input
* is available.
*
* IMPORTANT API CONTRACT:
* - `dst` MUST be the same stable buffer base across all calls for
* a single frame. The decoder tracks its own output position
* inside `dst` and requires it not to move between calls.
* - `dst_cap` MUST be large enough to hold the fully-decoded
* content of the current frame (the decoder does not support
* partial-output-then-resume semantics across a block boundary).
* - `*written` is set to the CUMULATIVE total bytes written into
* `dst` so far, NOT the delta for this call. If you need the
* per-call delta, subtract the previous value.
* - `*consumed` is per-call: how many `src` bytes were processed
* this call.
*
* Writing pattern:
* size_t total_written = 0;
* while (!done) {
* rc = vv_dstream_decompress_chunk(ds, chunk, chunk_len,
* dst, dst_cap, // stable
* &consumed, &written);
* total_written = written; // NOT += written
* ...
* }
*
* Returns VV_OK (0) if more input is needed, 1 if the frame ended
* successfully, or negative error code on failure. */
int vv_dstream_decompress_chunk(vv_dstream_t *ctx,
const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap,
size_t *consumed, size_t *written);
/* Destroy a decompression stream context and free all resources. */
void vv_dstream_destroy(vv_dstream_t *ctx);
/* ═══════════════════════════════════════════════════════════════
* INTERNAL HELPERS (shared across modules)
* */
/* XXH64 hash (simplified, for checksum) */
/* VAPTVUPT: vv_xxh64 aliased to zupt_xxh64 (avoid duplicate symbol) */
#define vv_xxh64 zupt_xxh64
uint64_t vv_xxh64(const void *data, size_t len, uint64_t seed);
/* Streaming XXH64: init + update + finalize for when the input isn't
* contiguous in memory. Must produce the same 64-bit hash as a
* single-shot vv_xxh64() over the concatenated input. */
typedef struct {
uint64_t v1, v2, v3, v4;
uint64_t total_len;
uint64_t seed;
uint8_t buf[32];
size_t buf_len;
} vv_xxh64_state_t;
void vv_xxh64_init(vv_xxh64_state_t *s, uint64_t seed);
void vv_xxh64_update(vv_xxh64_state_t *s, const void *data, size_t len);
uint64_t vv_xxh64_finalize(const vv_xxh64_state_t *s);
/* Hash function for matcher */
static inline uint32_t vv_hash4(const uint8_t *p) {
uint32_t v;
__builtin_memcpy(&v, p, 4);
memcpy(&v, p, 4);
return (v * 2654435761u) >> (32 - VV_HC_BITS);
}
/* Read/write little-endian helpers */
static inline uint16_t vv_read16(const uint8_t *p) {
uint16_t v; __builtin_memcpy(&v, p, 2); return v;
uint16_t v; memcpy(&v, p, 2); return v;
}
static inline uint32_t vv_read32(const uint8_t *p) {
uint32_t v; __builtin_memcpy(&v, p, 4); return v;
uint32_t v; memcpy(&v, p, 4); return v;
}
static inline void vv_write16(uint8_t *p, uint16_t v) {
__builtin_memcpy(p, &v, 2);
memcpy(p, &v, 2);
}
static inline void vv_write32(uint8_t *p, uint32_t v) {
__builtin_memcpy(p, &v, 4);
memcpy(p, &v, 4);
}
/* ═══════════════════════════════════════════════════════════════

View file

@ -1,9 +1,3 @@
/* VaptVupt codec — originally Apache-2.0 by Cristian Cezar Moisés
* Integrated into Zupt MIT License
* Copyright (c) 2026 Cristian Cezar Moisés
* SPDX-License-Identifier: MIT AND Apache-2.0
*/
/*
* VaptVupt Zupt Integration API
* SPDX-License-Identifier: GPL-3.0-or-later

View file

@ -1,9 +1,3 @@
/* VaptVupt codec — originally Apache-2.0 by Cristian Cezar Moisés
* Integrated into Zupt MIT License
* Copyright (c) 2026 Cristian Cezar Moisés
* SPDX-License-Identifier: MIT AND Apache-2.0
*/
/*
* VaptVupt tANS Entropy Codec (v2: sparse header + 4-way interleaved)
*
@ -103,15 +97,31 @@ vva_error_t vva_decode_ctx(const uint8_t *src, size_t src_len,
#define VVA_ML_CODES 36 /* Match length code count */
#define VVA_OF_CODES 27 /* Offset code count: 3 rep + 24 explicit */
#define VVA_LL_CODES 36 /* Literal-run length code count (covers 0-65536+) */
vva_error_t vva_encode_sequences(const uint8_t *tokens, size_t tok_len,
uint8_t *dst, size_t dst_cap, size_t *dst_len,
int off_bytes);
/* Format-v2 variant: encodes match-length codes using ml_base_v2
* (min_match=3). Used for 'T' tag blocks. Added v2.34.0. */
vva_error_t vva_encode_sequences_v2(const uint8_t *tokens, size_t tok_len,
uint8_t *dst, size_t dst_cap, size_t *dst_len,
int off_bytes);
vva_error_t vva_decode_sequences(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap, size_t *dst_len,
const uint8_t *dst_base);
/* Format-v2 variant: same wire payload as vva_decode_sequences but
* interprets match-length codes with a table shifted down by 1
* (min_match=3 instead of 4). Produced by tag 'T' (VV_ENTROPY_SEQ_V2)
* blocks; closes the ~10% binary-compression gap vs gzip-9. Added
* v2.33.0. */
vva_error_t vva_decode_sequences_v2(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap, size_t *dst_len,
const uint8_t *dst_base);
static inline size_t vva_bound(size_t src_len) {
/* Context model header can be up to ~10KB, seq coding adds 3 table headers */
return 12288 + (src_len * 15 + 7) / 8 + 16;

View file

@ -1,9 +1,3 @@
/* VaptVupt codec — originally Apache-2.0 by Cristian Cezar Moisés
* Integrated into Zupt MIT License
* Copyright (c) 2026 Cristian Cezar Moisés
* SPDX-License-Identifier: MIT AND Apache-2.0
*/
/*
* VaptVupt Canonical Huffman Codec
*

117
include/vv_platform.h Normal file
View file

@ -0,0 +1,117 @@
/*
* VaptVupt Cross-platform portability macros
*
* Provides unified abstractions for compiler intrinsics used throughout
* the codebase. Supports GCC, Clang, MSVC, and Intel compilers.
*
* SPDX-License-Identifier: GPL-3.0-or-later
*/
#ifndef VV_PLATFORM_H
#define VV_PLATFORM_H
#include <string.h>
#include <stdint.h>
/* ─── Branch prediction hints ─── */
#if defined(__GNUC__) || defined(__clang__)
#define VV_LIKELY(x) __builtin_expect(!!(x), 1)
#define VV_UNLIKELY(x) __builtin_expect(!!(x), 0)
#else
#define VV_LIKELY(x) (x)
#define VV_UNLIKELY(x) (x)
#endif
/* ─── Prefetch hint ─── */
#if defined(__GNUC__) || defined(__clang__)
#define VV_PREFETCH(p) __builtin_prefetch((p), 0, 1)
#define VV_PREFETCH_RW(p) __builtin_prefetch((p), 1, 1)
#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
#include <emmintrin.h>
#define VV_PREFETCH(p) _mm_prefetch((const char*)(p), _MM_HINT_T1)
#define VV_PREFETCH_RW(p) _mm_prefetch((const char*)(p), _MM_HINT_T1)
#else
#define VV_PREFETCH(p) ((void)0)
#define VV_PREFETCH_RW(p) ((void)0)
#endif
/* ─── Always-inline / never-inline ─── */
#if defined(__GNUC__) || defined(__clang__)
#define VV_ALWAYS_INLINE static inline __attribute__((always_inline))
#define VV_NOINLINE __attribute__((noinline))
#elif defined(_MSC_VER)
#define VV_ALWAYS_INLINE static __forceinline
#define VV_NOINLINE __declspec(noinline)
#else
#define VV_ALWAYS_INLINE static inline
#define VV_NOINLINE
#endif
/* ─── Unused parameter suppression ─── */
#if defined(__GNUC__) || defined(__clang__)
#define VV_UNUSED __attribute__((unused))
#else
#define VV_UNUSED
#endif
/* ─── Portable unaligned load/store via memcpy (compiler optimizes to single instr) ─── */
static inline uint16_t vv_load16(const void *p) {
uint16_t v; memcpy(&v, p, 2); return v;
}
static inline uint32_t vv_load32(const void *p) {
uint32_t v; memcpy(&v, p, 4); return v;
}
static inline uint64_t vv_load64(const void *p) {
uint64_t v; memcpy(&v, p, 8); return v;
}
static inline void vv_store16(void *p, uint16_t v) { memcpy(p, &v, 2); }
static inline void vv_store32(void *p, uint32_t v) { memcpy(p, &v, 4); }
static inline void vv_store64(void *p, uint64_t v) { memcpy(p, &v, 8); }
/* ─── Count trailing zeros (for hash/match optimization) ─── */
#if defined(__GNUC__) || defined(__clang__)
static inline int vv_ctz32(uint32_t x) { return __builtin_ctz(x); }
static inline int vv_ctz64(uint64_t x) { return __builtin_ctzll(x); }
#elif defined(_MSC_VER)
#include <intrin.h>
static inline int vv_ctz32(uint32_t x) {
unsigned long idx; _BitScanForward(&idx, x); return (int)idx;
}
static inline int vv_ctz64(uint64_t x) {
#if defined(_M_X64) || defined(_M_ARM64)
unsigned long idx; _BitScanForward64(&idx, x); return (int)idx;
#else
uint32_t lo = (uint32_t)x;
if (lo) return vv_ctz32(lo);
return 32 + vv_ctz32((uint32_t)(x >> 32));
#endif
}
#else
static inline int vv_ctz32(uint32_t x) {
int n = 0; while (!(x & 1)) { x >>= 1; n++; } return n;
}
static inline int vv_ctz64(uint64_t x) {
int n = 0; while (!(x & 1)) { x >>= 1; n++; } return n;
}
#endif
/* ─── SIMD capability detection macros ─── */
#if defined(__AVX2__)
#define VV_HAS_AVX2 1
#else
#define VV_HAS_AVX2 0
#endif
#if defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)
#define VV_HAS_SSE2 1
#else
#define VV_HAS_SSE2 0
#endif
#if defined(__aarch64__) && defined(__ARM_NEON)
#define VV_HAS_NEON 1
#else
#define VV_HAS_NEON 0
#endif
#endif /* VV_PLATFORM_H */

View file

@ -30,7 +30,7 @@
#define zupt_mkdir(p) mkdir(p, 0755)
#endif
#define ZUPT_VERSION_STRING "2.1.5"
#define ZUPT_VERSION_STRING "2.1.6"
#define ZUPT_FORMAT_MAJOR 1
#define ZUPT_FORMAT_MINOR 4
@ -380,4 +380,7 @@ void zupt_dedup_stats(const zupt_dedup_ctx_t *ctx,
int zupt_dedup_write_ref(FILE *out, uint64_t ref_offset,
uint32_t orig_size, uint64_t orig_checksum);
/* ─── Archive Info (read-only metadata inspection) ─── */
zupt_error_t zupt_archive_info(const char *path);
#endif /* ZUPT_H */