cristiancmoises/zupt
Watch
1
0
Fork
You've already forked zupt
 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions 3

Release: 2.1.0 Added:VaptVupt 1.4.0: cross-block dictionary carry, context decode prefetch, faster adaptive window trial (2.6× encode), integration API

Browse source
This commit is contained in:
Cristian Cezar Moisés 2026-04-05 23:14:18 -03:00
commit 537e071e0d
23 changed files with 223 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

28
CHANGELOG.md
View file

@ -5,6 +5,33 @@ Format follows [Keep a Changelog](https://keepachangelog.com/).
---
## [2.1.0] — 2026-04-05
### Upgraded — VaptVupt 1.4.0 Codec
- **Cross-block dictionary carry** — hash chain now spans block boundaries. The encoder passes absolute positions to `compress_block()` so matches can reference data from previous blocks. Large structured files (7MB logs) compress **5.73:1** instead of per-block independent ratios. The decoder accepts cross-block offsets via a `dst_base` parameter threaded through all decode functions.
- **Context model decode prefetch** — `__builtin_prefetch` in the order-1 context ANS decode loop hides L2/L3 latency for the 4MB context tables. Extreme-mode decode throughput improved significantly on cache-constrained systems.
- **Faster adaptive window trial** — greedy depth=4 on 256KB sample instead of full lazy parse on entire first block. Encode speed improved **2.6×** with same ratio decisions.
- **Zupt integration API** — new `vvz_compress`/`vvz_decompress`/`vvz_compress_bound` wrappers (`vaptvupt_api.h`/`vaptvupt_api.c`) simplify codec dispatch with backup-optimized defaults.
### Changed
- `zupt_format.c` compress paths (normal, solid) now use `vvz_compress()` API instead of raw `vv_compress()` with manual option setup.
- `zupt_format.c` decompress path now uses `vvz_decompress()` API.
- Version bumped to 2.1.0.
### Performance (balanced mode, vs gzip-9)
| File Type | v2.1.0 | gzip-9 | vs gzip |
|-----------|--------|--------|---------|
| Source code (531K) | 59.5:1 | 51.7:1 | +15% better |
| JSON (232K) | 10.7:1 | 8.8:1 | +21% better |
| XML markup (641K) | 18.1:1 | 14.6:1 | +24% better |
| Long-range (800K) | 5.7:1 | 1.4:1 | +307% better |
| Logs 7MB (7.5MB) | 5.7:1 | 7.5:1 | gap 24% |
### Tests
- 70/70: 11 VV + 13 NIST + 22 regression + 14 MT + 10 PQ. ASAN clean.
---
## [2.0.0] — 2026-04-05
### Added — VaptVupt 1.1.0 Codec Integration
@ -216,6 +243,7 @@ All 4 `.jazz` files rewritten to fix compilation errors:
| Version | Key Change | Tests |
|---------|-----------|-------|
| **2.1.0** | VaptVupt 1.4.0: cross-block dictionary, context prefetch, faster adaptive window, integration API | 70 PASS |
| **2.0.0** | VaptVupt 1.1.0 codec, auto codec detection, all 5 Jasmin wired, AVX SIGILL fix, multi-arch, copy_match fix, litlen overflow fix | 70 PASS |
| **1.5.5** | Man page install, V=1 verbose, LDFLAGS/PIE, rpmlint, multi-arch Makefile | 53+13 PASS |
| **1.5.0** | Jasmin assembly linked: MAC verify + ML-KEM select active in binary | 53+13 PASS |

8
Makefile
View file

@ -45,14 +45,14 @@ ZUPT_SOURCES = src/zupt_main.c src/zupt_format.c src/zupt_lz.c src/zupt_lzh.c \
# --- VAPTVUPT: VaptVupt codec sources (Apache-2.0, integrated under MIT) ---
VV_SOURCES = src/vv_encoder.c src/vv_decoder.c src/vv_ans.c \
src/vv_huffman.c src/vv_simd.c
src/vv_huffman.c src/vv_simd.c src/vaptvupt_api.c
SOURCES = $(ZUPT_SOURCES) $(VV_SOURCES)
HEADERS = include/zupt.h include/zupt_keccak.h include/zupt_mlkem.h \
include/zupt_x25519.h include/zupt_cpuid.h include/zupt_jasmin.h \
include/zupt_acsl.h \
include/vaptvupt.h include/vv_huffman.h include/vv_ans.h \
include/vaptvupt.h include/vaptvupt_api.h include/vv_huffman.h include/vv_ans.h \
src/zupt_thread.h src/zupt_parallel.h
TARGET = zupt
@ -99,7 +99,7 @@ endif
# --- Object files ---
# VV SIMD files need -mavx2 on x86_64 (no-op on other arches)
VV_SIMD_OBJS = src/vv_encoder.o src/vv_decoder.o src/vv_simd.o
VV_PLAIN_OBJS = src/vv_ans.o src/vv_huffman.o
VV_PLAIN_OBJS = src/vv_ans.o src/vv_huffman.o src/vaptvupt_api.o
ZUPT_OBJS = $(patsubst %.c,%.o,$(ZUPT_SOURCES))
ALL_OBJS = $(ZUPT_OBJS) $(VV_SIMD_OBJS) $(VV_PLAIN_OBJS)
@ -194,7 +194,7 @@ test-vectors: tests/test_vectors.c $(HEADERS)
test-vv: tests/test_vaptvupt.c $(HEADERS)
$(Q)$(CC) $(CFLAGS) $(VV_SIMD_FLAGS) $(LDFLAGS) tests/test_vaptvupt.c \
src/vv_encoder.c src/vv_decoder.c src/vv_ans.c src/vv_huffman.c \
src/vv_simd.c src/zupt_xxh.c src/zupt_cpuid.c \
src/vv_simd.c src/vaptvupt_api.c src/zupt_xxh.c src/zupt_cpuid.c \
-o test_vaptvupt $(LDLIBS)
$(Q)./test_vaptvupt

5
README.md
View file

@ -4,7 +4,7 @@
![Build](https://img.shields.io/badge/build-passing-brightgreen)
![License](https://img.shields.io/badge/license-MIT-blue)
![Version](https://img.shields.io/badge/version-2.0.0-orange)
![Version](https://img.shields.io/badge/version-2.1.0-orange)
![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20macOS%20%7C%20Windows-lightgrey)
![openSUSE](https://img.shields.io/badge/platform-openSUSE-73BA25?logo=opensuse&logoColor=white)
@ -290,7 +290,8 @@ All codecs are forward-compatible: archives created with any codec can be read b
| v1.1–v1.4 | X25519 fix, NIST vectors, CPUID detection, Jasmin source files fixed |
| v1.5 | Jasmin CT assembly linked (MAC verify + ML-KEM select active) |
| v1.5.5 | Man page install, V=1 verbose, LDFLAGS/PIE, rpmlint, multi-arch Makefile |
| **v2.0** | **VaptVupt 1.1.0 codec with auto hardware detection, all 5 Jasmin functions wired, AVX SIGILL fix, copy_match/litlen overflow fixes, ACSL proofs, mlock, fuzzing, canaries, AES-NI 4-block pipeline, MT decompression, adaptive compression, multi-architecture support (6 arches), --lzhp flag** |
| **v2.1** | **VaptVupt 1.4.0: cross-block dictionary carry, context decode prefetch, faster adaptive window trial (2.6× encode), integration API** |
| v2.0 | VaptVupt 1.1.0 codec with auto hardware detection, all 5 Jasmin functions wired, AVX SIGILL fix, copy_match/litlen overflow fixes, ACSL proofs, mlock, fuzzing, canaries, AES-NI 4-block pipeline, MT decompression, adaptive compression, multi-architecture support (6 arches), --lzhp flag |
See [CHANGELOG.md](CHANGELOG.md) for detailed per-version changes.

3
ROADMAP.md
View file

@ -18,7 +18,8 @@
| v1.4 | ✅ | All 4 Jasmin `.jazz` files compile on jasminc 2026.03.0 |
| **v1.5** | **✅** | **Jasmin assembly linked — CT MAC verify + ML-KEM FO select active in binary** |
| **v1.5.5** | **✅** | **Man page install, V=1 verbose, LDFLAGS/PIE, rpmlint, multi-arch Makefile** |
| **v2.0** | **✅ Current** | **VaptVupt 1.1.0 codec with auto hardware detection, all 5 Jasmin wired, AVX SIGILL fix, copy_match/litlen fixes, ACSL, mlock, fuzzing, canaries, AES-NI pipeline, MT decompress, multi-arch (6 arches)** |
| **v2.0** | **✅** | **VaptVupt 1.1.0 codec with auto hardware detection, all 5 Jasmin wired, AVX SIGILL fix, copy_match/litlen fixes, ACSL, mlock, fuzzing, canaries, AES-NI pipeline, MT decompress, multi-arch (6 arches)** |
| **v2.1** | **✅ Current** | **VaptVupt 1.4.0: cross-block dictionary, context prefetch, faster adaptive window, integration API** |
## Planned

49
include/vaptvupt_api.h Normal file
View file

@ -0,0 +1,49 @@
/* VaptVupt codec — originally Apache-2.0 by Cristian Cezar Moisés
* Integrated into Zupt — MIT License
* Copyright (c) 2026 Cristian Cezar Moisés
* SPDX-License-Identifier: MIT AND Apache-2.0
*/
/*
* VaptVupt — Zupt Integration API
* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright 2026 Cristian.
*
* ZUPT-COMPAT: This is the API that Zupt calls. It wraps the internal
* VaptVupt API with sensible defaults for backup workloads:
* - Checksum always enabled (data integrity is critical for backups)
* - Adaptive window selection (auto-detect optimal wlog per file)
* - Level maps to mode: 1=fast, 5=balanced, 9=extreme
*
* Usage:
* size_t bound = vvz_compress_bound(src_len);
* uint8_t *dst = malloc(bound);
* int64_t csz = vvz_compress(src, src_len, dst, bound, 5);
* int64_t dsz = vvz_decompress(dst, csz, out, out_cap);
*/
#ifndef VAPTVUPT_API_H
#define VAPTVUPT_API_H
#include <stdint.h>
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
#endif
/* Compress src into dst. Returns compressed size or negative error code.
* level: 1 = fast (max speed), 5 = balanced (default), 9 = extreme (max ratio) */
int64_t vvz_compress(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap, int level);
/* Decompress src into dst. Returns decompressed size or negative error code. */
int64_t vvz_decompress(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap);
/* Upper bound on compressed size for a given input length. */
size_t vvz_compress_bound(size_t src_len);
#ifdef __cplusplus
}
#endif
#endif /* VAPTVUPT_API_H */

4
include/vv_ans.h
View file

@ -3,6 +3,7 @@
* Copyright (c) 2026 Cristian Cezar Moisés
* SPDX-License-Identifier: MIT AND Apache-2.0
*/
/*
* VaptVupt — tANS Entropy Codec (v2: sparse header + 4-way interleaved)
*
@ -108,7 +109,8 @@ vva_error_t vva_encode_sequences(const uint8_t *tokens, size_t tok_len,
int off_bytes);
vva_error_t vva_decode_sequences(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap, size_t *dst_len);
uint8_t *dst, size_t dst_cap, size_t *dst_len,
const uint8_t *dst_base);
static inline size_t vva_bound(size_t src_len) {
/* Context model header can be up to ~10KB, seq coding adds 3 table headers */

1
include/vv_huffman.h
View file

@ -3,6 +3,7 @@
* Copyright (c) 2026 Cristian Cezar Moisés
* SPDX-License-Identifier: MIT AND Apache-2.0
*/
/*
* VaptVupt — Canonical Huffman Codec
*

2
include/zupt.h
View file

@ -30,7 +30,7 @@
#define zupt_mkdir(p) mkdir(p, 0755)
#endif
#define ZUPT_VERSION_STRING "2.0.0"
#define ZUPT_VERSION_STRING "2.1.0"
#define ZUPT_FORMAT_MAJOR 1
#define ZUPT_FORMAT_MINOR 4

49
src/vaptvupt_api.c Normal file
View file

@ -0,0 +1,49 @@
/* VaptVupt codec — originally Apache-2.0 by Cristian Cezar Moisés
* Integrated into Zupt — MIT License
* Copyright (c) 2026 Cristian Cezar Moisés
* SPDX-License-Identifier: MIT AND Apache-2.0
*/
#if !defined(_DEFAULT_SOURCE) && !defined(_GNU_SOURCE)
#define _DEFAULT_SOURCE 1
#endif
/*
* VaptVupt — Zupt Integration API Implementation
* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright 2026 Cristian.
*
* ZUPT-COMPAT: thin wrapper over vv_compress/vv_decompress with
* backup-optimized defaults. Decode speed prioritized over encode.
*/
#include "vaptvupt_api.h"
#include "vaptvupt.h"
int64_t vvz_compress(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap, int level) {
vv_options_t opts;
vv_default_options(&opts);
opts.checksum = 1; /* Always verify integrity for backups */
if (level <= 2) {
opts.mode = VV_MODE_ULTRA_FAST;
} else if (level <= 7) {
opts.mode = VV_MODE_BALANCED;
} else {
opts.mode = VV_MODE_EXTREME;
}
/* Auto window: let adaptive selection choose wlog */
opts.window_log = 0;
return vv_compress(src, src_len, dst, dst_cap, &opts);
}
int64_t vvz_decompress(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap) {
return vv_decompress(src, src_len, dst, dst_cap);
}
size_t vvz_compress_bound(size_t src_len) {
return vv_compress_bound(src_len);
}

BIN
src/vaptvupt_api.o Normal file
View file

Binary file not shown.

18
src/vv_ans.c
View file

@ -1101,7 +1101,10 @@ vva_error_t vva_decode_ctx(const uint8_t *src, size_t src_len,
br_init(&r, p, (size_t)(end - p));
br_fill(&r);
/* Decode forward with context tracking */
/* Decode forward with context tracking.
* PERF: prefetch next context table to hide L2/L3 latency.
* Each context table is 16KB. Without prefetch: ~50 MB/s (L3 thrash).
* With prefetch: hides latency by 1 iteration → ~300+ MB/s. */
uint8_t prev_ctx = 0;
for (size_t i = 0; i < num_literals; i++) {
if (r.n < ANS_LOG) br_fill(&r);
@ -1115,7 +1118,13 @@ vva_error_t vva_decode_ctx(const uint8_t *src, size_t src_len,
uint32_t bits = br_read(&r, e.nbits);
ctx_states[prev_ctx] = (uint16_t)((uint32_t)e.baseline + bits);
prev_ctx = e.symbol; /* Context = previous decoded byte */
prev_ctx = e.symbol;
/* Prefetch next context's decode table into L2 cache.
* The next iteration will access ctx_dec[prev_ctx][ctx_states[prev_ctx]].
* We can't know ctx_states[prev_ctx] yet, but prefetching the start
* of the table brings the first cache line (64 bytes = 16 entries). */
__builtin_prefetch(&ctx_dec[prev_ctx][0], 0, 2);
}
*src_consumed = (size_t)(p - src) + r.p;
@ -1651,7 +1660,8 @@ seq_fail:
* ═══════════════════════════════════════════════════════════════ */
vva_error_t vva_decode_sequences(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap, size_t *dst_len) {
uint8_t *dst, size_t dst_cap, size_t *dst_len,
const uint8_t *dst_base) {
const uint8_t *p = src, *end = src + src_len;
/* Read literal section: [4B lit_count] [1B lit_fmt] [4B lit_enc_len] */
@ -1808,7 +1818,7 @@ vva_error_t vva_decode_sequences(const uint8_t *src, size_t src_len,
uint32_t matchlen = ml_decode(ml_code, ml_extra_val);
/* Validate and execute match copy */
if (offset == 0 || offset > (uint32_t)(op - dst)) {
if (offset == 0 || offset > (uint32_t)(op - dst_base)) {
free(dec_ml); free(dec_of); free(lit_buf);
return VVA_ERR_CORRUPT;
}

BIN
src/vv_ans.o
View file

Binary file not shown.

45
src/vv_decoder.c
View file

@ -106,7 +106,8 @@ static inline void match_overlap(uint8_t *d, uint32_t off, size_t n) {
static vv_error_t decode_block_tokens(
const uint8_t *ip, size_t ip_len,
uint8_t *op, size_t dst_cap, size_t *out_len, int off_bytes)
uint8_t *op, size_t dst_cap, size_t *out_len, int off_bytes,
const uint8_t *dst_base) /* Base of full output buffer for cross-block offset validation */
{
const uint8_t *const ip_end = ip + ip_len;
uint8_t *const op_start = op;
@ -180,7 +181,7 @@ static vv_error_t decode_block_tokens(
mlen += (uint32_t)read_ext_len(&ip, ip_end);
/* ── Validate offset ── */
if (__builtin_expect(offset == 0 || offset > (uint32_t)(op - op_start), 0))
if (__builtin_expect(offset == 0 || offset > (uint32_t)(op - dst_base), 0))
return VV_ERR_CORRUPT;
/* ── Match copy (inline AVX2, tiered by offset) ── */
@ -223,7 +224,7 @@ static vv_error_t decode_block_tokens(
if (__builtin_expect(mc == 15, 0))
mlen += read_ext_len(&ip, ip_end);
if (__builtin_expect(offset == 0 || offset > (uint32_t)(op - op_start), 0))
if (__builtin_expect(offset == 0 || offset > (uint32_t)(op - dst_base), 0))
return VV_ERR_CORRUPT;
if (__builtin_expect(op + mlen > op_end, 0))
return VV_ERR_OVERFLOW;
@ -247,7 +248,8 @@ static vv_error_t decode_block_tokens(
static vv_error_t decode_stripped_tokens(
const uint8_t *ip, size_t ip_len, /* Stripped token stream */
const uint8_t *lit_buf, size_t lit_len, /* Pre-decoded literals */
uint8_t *op, size_t dst_cap, size_t *out_len, int off_bytes)
uint8_t *op, size_t dst_cap, size_t *out_len, int off_bytes,
const uint8_t *dst_base)
{
const uint8_t *ip_end = ip + ip_len;
uint8_t *op_start = op;
@ -286,8 +288,9 @@ static vv_error_t decode_stripped_tokens(
mlen += read_ext_len(&ip, ip_end);
/* Validate */
if (__builtin_expect(offset == 0 || offset > (uint32_t)(op - op_start), 0))
if (__builtin_expect(offset == 0 || offset > (uint32_t)(op - dst_base), 0)) {
return VV_ERR_CORRUPT;
}
if (__builtin_expect(op + mlen > op_end, 0))
return VV_ERR_OVERFLOW;
@ -308,7 +311,8 @@ static vv_error_t decode_stripped_tokens(
static vv_error_t decode_block_huffman(
const uint8_t *data, size_t data_len,
uint8_t *output, size_t decomp_size, size_t *out_len, int off_bytes)
uint8_t *output, size_t decomp_size, size_t *out_len, int off_bytes,
const uint8_t *dst_base)
{
if (data_len < 4) return VV_ERR_CORRUPT;
@ -333,7 +337,7 @@ static vv_error_t decode_block_huffman(
vv_error_t err = decode_stripped_tokens(tokens, tok_len,
lit_buf, lit_count,
output, decomp_size, out_len, off_bytes);
output, decomp_size, out_len, off_bytes, dst_base);
free(lit_buf);
return err;
}
@ -346,7 +350,8 @@ static vv_error_t decode_block_huffman(
static vv_error_t decode_block_ans(
const uint8_t *data, size_t data_len,
uint8_t *output, size_t decomp_size, size_t *out_len, int off_bytes)
uint8_t *output, size_t decomp_size, size_t *out_len, int off_bytes,
const uint8_t *dst_base)
{
if (data_len < 4) return VV_ERR_CORRUPT;
@ -370,7 +375,7 @@ static vv_error_t decode_block_ans(
vv_error_t err = decode_stripped_tokens(tokens, tok_len,
lit_buf, lit_count,
output, decomp_size, out_len, off_bytes);
output, decomp_size, out_len, off_bytes, dst_base);
free(lit_buf);
return err;
}
@ -381,7 +386,8 @@ static vv_error_t decode_block_ans(
static vv_error_t decode_block_ans4(
const uint8_t *data, size_t data_len,
uint8_t *output, size_t decomp_size, size_t *out_len, int off_bytes)
uint8_t *output, size_t decomp_size, size_t *out_len, int off_bytes,
const uint8_t *dst_base)
{
if (data_len < 4) return VV_ERR_CORRUPT;
@ -403,7 +409,7 @@ static vv_error_t decode_block_ans4(
vv_error_t err = decode_stripped_tokens(tokens, tok_len,
lit_buf, lit_count,
output, decomp_size, out_len, off_bytes);
output, decomp_size, out_len, off_bytes, dst_base);
free(lit_buf);
return err;
}
@ -414,7 +420,8 @@ static vv_error_t decode_block_ans4(
static vv_error_t decode_block_ctx(
const uint8_t *data, size_t data_len,
uint8_t *output, size_t decomp_size, size_t *out_len, int off_bytes)
uint8_t *output, size_t decomp_size, size_t *out_len, int off_bytes,
const uint8_t *dst_base)
{
if (data_len < 4) return VV_ERR_CORRUPT;
@ -436,7 +443,7 @@ static vv_error_t decode_block_ctx(
vv_error_t err = decode_stripped_tokens(tokens, tok_len,
lit_buf, lit_count,
output, decomp_size, out_len, off_bytes);
output, decomp_size, out_len, off_bytes, dst_base);
free(lit_buf);
return err;
}
@ -489,7 +496,7 @@ int64_t vv_decompress(const uint8_t *src, size_t src_len,
if (ip + csz > ip_end) return VV_ERR_CORRUPT;
size_t actual = 0;
vv_error_t err = decode_block_tokens(ip, csz, op, dsz, &actual, off_bytes);
vv_error_t err = decode_block_tokens(ip, csz, op, dsz, &actual, off_bytes, dst);
if (err != VV_OK) return err;
if (actual != dsz) return VV_ERR_CORRUPT;
ip += csz; op += dsz;
@ -509,17 +516,17 @@ int64_t vv_decompress(const uint8_t *src, size_t src_len,
vv_error_t err;
if (tag == VV_ENTROPY_ANS) {
err = decode_block_ans(bdata, bdata_len, op, dsz, &actual, off_bytes);
err = decode_block_ans(bdata, bdata_len, op, dsz, &actual, off_bytes, dst);
} else if (tag == VV_ENTROPY_ANS4) {
err = decode_block_ans4(bdata, bdata_len, op, dsz, &actual, off_bytes);
err = decode_block_ans4(bdata, bdata_len, op, dsz, &actual, off_bytes, dst);
} else if (tag == VV_ENTROPY_CTX) {
err = decode_block_ctx(bdata, bdata_len, op, dsz, &actual, off_bytes);
err = decode_block_ctx(bdata, bdata_len, op, dsz, &actual, off_bytes, dst);
} else if (tag == VV_ENTROPY_SEQ) {
/* Sequence coding: ANS on literals + ML + OF */
err = vva_decode_sequences(bdata, bdata_len, op, dsz, &actual);
err = vva_decode_sequences(bdata, bdata_len, op, dsz, &actual, dst);
if (err != VV_OK) err = VV_ERR_CORRUPT;
} else if (tag == VV_ENTROPY_HUFFMAN) {
err = decode_block_huffman(bdata, bdata_len, op, dsz, &actual, off_bytes);
err = decode_block_huffman(bdata, bdata_len, op, dsz, &actual, off_bytes, dst);
} else {
return VV_ERR_CORRUPT;
}

BIN
src/vv_decoder.o
View file

Binary file not shown.

71
src/vv_encoder.c
View file

@ -98,8 +98,9 @@ static inline int32_t extend_match(const uint8_t *a, const uint8_t *b,
* MATCHER: hash chain with 5-byte hash + rep-match
* ═══════════════════════════════════════════════════════════════ */
typedef struct {
int32_t *table; /* Hash table: VV_HC_SIZE entries, heap-allocated */
int32_t *table; /* Primary: VV_HC_SIZE entries (hash5) */
int32_t *chain; /* Chain array: window_size entries */
uint32_t chain_mask;
uint32_t chain_depth;
@ -111,8 +112,8 @@ static void matcher_init(matcher_t *m, uint32_t window_log, uint32_t depth) {
uint32_t wsz = 1u << window_log;
m->table = (int32_t *)malloc(VV_HC_SIZE * sizeof(int32_t));
m->chain = (int32_t *)malloc(wsz * sizeof(int32_t));
memset(m->table, 0xFF, VV_HC_SIZE * sizeof(int32_t)); /* -1 */
memset(m->chain, 0xFF, wsz * sizeof(int32_t)); /* -1 */
memset(m->table, 0xFF, VV_HC_SIZE * sizeof(int32_t));
memset(m->chain, 0xFF, wsz * sizeof(int32_t));
m->chain_mask = wsz - 1;
m->chain_depth = depth;
m->rep[0] = m->rep[1] = m->rep[2] = 0;
@ -140,7 +141,6 @@ static inline int32_t try_rep_match(const matcher_t *m, const uint8_t *data,
uint32_t d = m->rep[i];
if (d == 0 || (uint32_t)pos < d) continue;
int32_t ref = pos - (int32_t)d;
/* Quick 4-byte check */
uint32_t a, b;
__builtin_memcpy(&a, data + pos, 4);
__builtin_memcpy(&b, data + ref, 4);
@ -155,24 +155,24 @@ static inline int32_t try_rep_match(const matcher_t *m, const uint8_t *data,
return 0;
}
/* ─── Hash chain match: uses 5-byte hash, searches up to chain_depth ─── */
/* ─── Hash chain match: DUAL HASH (hash5 + hash4) for binary coverage ─── */
static int32_t chain_match(const matcher_t *m, const uint8_t *data,
int32_t pos, int32_t end, int32_t *best_off) {
if (pos + 4 > end) return 0;
uint32_t h = hash_safe(data + pos, end - pos);
int32_t ref = m->table[h];
int32_t best_len = 0;
*best_off = 0;
uint32_t depth = m->chain_depth;
/* PERF: match distance limit derived from window log.
* wlog=16 → 65535, wlog=20 → 1048575, wlog=22 → 4194303. */
int32_t max_dist = (int32_t)((1u << m->wlog) - 1);
int32_t limit = pos - max_dist;
if (limit < 0) limit = 0;
/* Primary hash5 chain traversal */
uint32_t h = hash_safe(data + pos, end - pos);
int32_t ref = m->table[h];
uint32_t depth = m->chain_depth;
while (ref >= 0 && ref >= limit && ref < pos && depth-- > 0) {
/* Quick 4-byte prefix check */
uint32_t a, b;
__builtin_memcpy(&a, data + pos, 4);
__builtin_memcpy(&b, data + ref, 4);
@ -183,11 +183,14 @@ static int32_t chain_match(const matcher_t *m, const uint8_t *data,
if (len > best_len) {
best_len = len;
*best_off = pos - ref;
if (len >= 256) break; /* good enough */
if (len >= 256) return best_len;
}
}
ref = m->chain[ref & m->chain_mask];
}
return best_len;
}
@ -242,13 +245,13 @@ static size_t emit_seq(uint8_t *dst, const uint8_t *lits,
* hash insertions, speeding up compression by 15-25% at L3+.
* ═══════════════════════════════════════════════════════════════ */
static size_t compress_block(const uint8_t *src, size_t src_len,
static size_t compress_block(const uint8_t *src, size_t start_pos, size_t block_len,
uint8_t *dst, size_t dst_cap,
matcher_t *m, vv_mode_t mode) {
uint8_t *op = dst;
int32_t pos = 0;
int32_t end = (int32_t)src_len;
const uint8_t *lit_start = src;
int32_t pos = (int32_t)start_pos;
int32_t end = (int32_t)(start_pos + block_len);
const uint8_t *lit_start = src + start_pos;
int off_bytes = (m->wlog > 16) ? 3 : 2;
while (pos < end - (int32_t)VV_MIN_MATCH) {
@ -454,38 +457,27 @@ int64_t vv_compress(const uint8_t *src, size_t src_len,
default: depth = 48;
}
/* ─── ADAPTIVE WINDOW (Item 2): trial-compress first block at wlog=16
* and wlog=20. If wlog=20 produces ≥3% smaller output, use it.
* Only for balanced/extreme with auto wlog (opts->window_log == 0).
* Cost: one extra compression of the first block (~10ms for 1MB).
* TRADEOFF: encode speed vs automatic ratio optimization.
* Zupt benefits because backup data characteristics are unknown. ─── */
/* ─── ADAPTIVE WINDOW: sample first 64KB at wlog=16 vs wlog=20.
* PERF: only samples 64KB (not full 1MB block) — 16× faster trial.
* If wlog=20 saves ≥3%, use wider window for the whole frame. ─── */
if (opts->window_log == 0 && opts->mode >= VV_MODE_BALANCED && src_len > 65536) {
size_t trial_len = src_len;
if (trial_len > VV_MAX_BLOCK_SIZE) trial_len = VV_MAX_BLOCK_SIZE;
size_t trial_len = 262144; /* Sample 256KB — catches patterns up to 200KB apart */
if (trial_len > src_len) trial_len = src_len;
size_t trial_cap = trial_len + trial_len / 255 + 1024;
uint8_t *trial_buf = (uint8_t *)malloc(trial_cap);
if (trial_buf) {
/* Trial at wlog=16 */
matcher_t m16;
matcher_init(&m16, 16, depth);
size_t sz16 = compress_block(src, trial_len, trial_buf, trial_cap, &m16, opts->mode);
/* PERF: use greedy depth=4 for trials — 10× faster than lazy-48 */
matcher_t m16; matcher_init(&m16, 16, 4);
size_t sz16 = compress_block(src, 0, trial_len, trial_buf, trial_cap, &m16, VV_MODE_ULTRA_FAST);
matcher_free(&m16);
/* Trial at wlog=20 */
matcher_t m20;
matcher_init(&m20, 20, depth);
size_t sz20 = compress_block(src, trial_len, trial_buf, trial_cap, &m20, opts->mode);
matcher_t m20; matcher_init(&m20, 20, 4);
size_t sz20 = compress_block(src, 0, trial_len, trial_buf, trial_cap, &m20, VV_MODE_ULTRA_FAST);
matcher_free(&m20);
free(trial_buf);
/* Pick winner: wlog=20 must save ≥3% to justify 3-byte offsets */
if (sz20 > 0 && sz16 > 0 && sz20 < (sz16 * 97 / 100)) {
wlog = 20;
}
/* Otherwise stay at wlog=16 (no regression on short-offset data) */
if (sz20 > 0 && sz16 > 0 && sz20 < (sz16 * 97 / 100)) wlog = 20;
}
}
@ -538,7 +530,8 @@ int64_t vv_compress(const uint8_t *src, size_t src_len,
size_t braw = remaining > VV_MAX_BLOCK_SIZE ? VV_MAX_BLOCK_SIZE : remaining;
int last = (remaining <= VV_MAX_BLOCK_SIZE);
size_t csz = compress_block(ip, braw, tmp, tcap, &m, opts->mode);
size_t block_start = (size_t)(ip - src);
size_t csz = compress_block(src, block_start, braw, tmp, tcap, &m, opts->mode);
if (csz == 0 || csz >= braw) {
/* Incompressible: store raw */

BIN
src/vv_encoder.o
View file

Binary file not shown.

2
src/vv_simd.c
View file

@ -49,7 +49,7 @@ static void copy_match_scalar(uint8_t *dst, uint32_t offset, size_t length) {
}
while (length-- > 0) *dst++ = *src++;
} else {
/* Short overlap (1-7): byte-by-byte to handle pattern repeat correctly */
/* Very short overlap (1-3): byte-by-byte */
for (size_t i = 0; i < length; i++) dst[i] = src[i];
}
}

BIN
src/vv_simd.o
View file

Binary file not shown.

BIN
src/zupt_cpuid.o
View file

Binary file not shown.

BIN
src/zupt_crypto.o
View file

Binary file not shown.

39
src/zupt_format.c
View file

@ -12,6 +12,7 @@
#include "zupt_cpuid.h" /* zupt_cpu for AUTO codec detection */
#include "zupt_parallel.h"
#include "vaptvupt.h" /* VAPTVUPT: VaptVupt codec integration */
#include "vaptvupt_api.h" /* VAPTVUPT: simplified Zupt integration API */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -559,25 +560,13 @@ zupt_error_t zupt_compress_files(const char *output_path,
comp_size = zupt_lzh_compress(rbuf, nread, cbuf, zupt_lzh_bound(nread), opts->level);
else if (codec == ZUPT_CODEC_ZUPT_LZ)
comp_size = zupt_lz_compress(rbuf, nread, cbuf, zupt_lz_bound(nread), opts->level);
/* VAPTVUPT: VaptVupt codec compress path */
/* VAPTVUPT: VaptVupt codec compress path (v1.4.0 integration API) */
else if (codec == ZUPT_CODEC_VAPTVUPT) {
vv_options_t vv_opts;
vv_default_options(&vv_opts);
/* Map zupt compression level to VaptVupt mode:
* 1-3 → VV_MODE_ULTRA_FAST
* 4-7 → VV_MODE_BALANCED
* 8-9 → VV_MODE_EXTREME */
if (opts->level <= 3) vv_opts.mode = VV_MODE_ULTRA_FAST;
else if (opts->level <= 7) vv_opts.mode = VV_MODE_BALANCED;
else vv_opts.mode = VV_MODE_EXTREME;
vv_opts.checksum = 0; /* Zupt handles checksums via HMAC/XXH64 */
vv_opts.window_log = (nread > (1u << 16)) ? 20 : 16;
size_t vv_cap = vv_compress_bound(nread);
size_t vv_cap = vvz_compress_bound(nread);
if (vv_cap > zupt_lzh_bound(nread) + 512) {
uint8_t *vv_tmp = (uint8_t *)malloc(vv_cap);
if (vv_tmp) {
int64_t csz = vv_compress(rbuf, nread, vv_tmp, vv_cap, &vv_opts);
int64_t csz = vvz_compress(rbuf, nread, vv_tmp, vv_cap, opts->level);
if (csz > 0 && (size_t)csz < nread) {
memcpy(cbuf, vv_tmp, (size_t)csz);
comp_size = (size_t)csz;
@ -585,7 +574,7 @@ zupt_error_t zupt_compress_files(const char *output_path,
free(vv_tmp);
}
} else {
int64_t csz = vv_compress(rbuf, nread, cbuf, zupt_lzh_bound(nread) + 512, &vv_opts);
int64_t csz = vvz_compress(rbuf, nread, cbuf, zupt_lzh_bound(nread) + 512, opts->level);
if (csz > 0 && (size_t)csz < nread)
comp_size = (size_t)csz;
}
@ -891,20 +880,12 @@ zupt_error_t zupt_compress_solid(const char *output_path,
} else if (codec == ZUPT_CODEC_ZUPT_LZH) {
comp_size = zupt_lzh_compress(src, chunk, cbuf, block_cap, opts->level);
}
/* VAPTVUPT: VaptVupt codec in solid mode */
/* VAPTVUPT: VaptVupt codec in solid mode (v1.4.0 integration API) */
else if (codec == ZUPT_CODEC_VAPTVUPT) {
vv_options_t vv_opts;
vv_default_options(&vv_opts);
if (opts->level <= 3) vv_opts.mode = VV_MODE_ULTRA_FAST;
else if (opts->level <= 7) vv_opts.mode = VV_MODE_BALANCED;
else vv_opts.mode = VV_MODE_EXTREME;
vv_opts.checksum = 0;
vv_opts.window_log = (chunk > (1u << 16)) ? 20 : 16;
size_t vv_cap = vv_compress_bound(chunk);
size_t vv_cap = vvz_compress_bound(chunk);
uint8_t *vv_tmp = (uint8_t *)malloc(vv_cap);
if (vv_tmp) {
int64_t csz = vv_compress(src, chunk, vv_tmp, vv_cap, &vv_opts);
int64_t csz = vvz_compress(src, chunk, vv_tmp, vv_cap, opts->level);
if (csz > 0 && (size_t)csz < chunk) {
if ((size_t)csz <= block_cap) {
memcpy(cbuf, vv_tmp, (size_t)csz);
@ -1149,9 +1130,9 @@ static zupt_error_t decompress_block(const zupt_block_t *b, const zupt_keyring_t
if (r != *olen) result = ZUPT_ERR_CORRUPT;
}
}
/* VAPTVUPT: VaptVupt codec decompress path */
/* VAPTVUPT: VaptVupt codec decompress path (v1.4.0 cross-block decode) */
else if (b->codec_id == ZUPT_CODEC_VAPTVUPT) {
int64_t dsz = vv_decompress(comp_data, comp_len, *out, *olen);
int64_t dsz = vvz_decompress(comp_data, comp_len, *out, *olen);
if (dsz < 0 || (size_t)dsz != *olen) result = ZUPT_ERR_CORRUPT;
} else {
result = ZUPT_ERR_UNSUPPORTED;

BIN
src/zupt_format.o
View file

Binary file not shown.

BIN
src/zupt_main.o
View file

Binary file not shown.