Release v2.0.0: VaptVupt codec integration, auto codec detection, Jasmin and VaptVupt fixes, multi-arch & performance enhancements, security hardening, ACSL annotations
This commit is contained in:
Cristian Cezar Moisés
parent
0e8e3b11c6
commit
c42251f0a2
15 changed files with 380 additions and 407 deletions
308
.github/workflows/ci.yml
vendored
308
.github/workflows/ci.yml
vendored
|
|
@ -1,308 +0,0 @@
|
|||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master]
|
||||
tags: ['v*']
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
|
||||
jobs:
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Linux x86_64 — GCC + Clang, full test suite
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
build-linux:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
compiler: [gcc, clang]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Build
|
||||
run: make CC=${{ matrix.compiler }}
|
||||
|
||||
- name: Verify Jasmin symbols (x86_64)
|
||||
run: |
|
||||
nm zupt | grep -q "T zupt_mac_verify_ct" || exit 1
|
||||
nm zupt | grep -q "T zupt_ct_select_32" || exit 1
|
||||
nm zupt | grep -q "T zupt_fe_cswap" || exit 1
|
||||
nm zupt | grep -q "T zupt_aes256_blk" || exit 1
|
||||
nm zupt | grep -q "T zupt_aes256_ctr4" || exit 1
|
||||
echo "All 5 Jasmin symbols linked"
|
||||
|
||||
- name: NIST/RFC test vectors (13 tests)
|
||||
run: make test-vectors CC=${{ matrix.compiler }} && ./test_vectors
|
||||
|
||||
- name: VaptVupt unit tests (11 tests)
|
||||
run: make test-vv CC=${{ matrix.compiler }}
|
||||
|
||||
- name: Regression tests (22 tests)
|
||||
run: bash tests/regression.sh
|
||||
|
||||
- name: Multi-threaded tests (14 tests)
|
||||
run: bash tests/test_threaded.sh
|
||||
|
||||
- name: Post-quantum tests (10 tests)
|
||||
run: bash tests/test_pq.sh ./zupt
|
||||
|
||||
- name: Benchmark
|
||||
run: ./zupt bench --compare
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Linux x86_64 — ASAN + UBSan
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
build-asan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Build with ASAN + UBSan
|
||||
run: make test-asan
|
||||
|
||||
- name: ASAN smoke test (all codecs + encryption + PQ)
|
||||
run: |
|
||||
T=$(mktemp -d)
|
||||
mkdir -p "$T/data/sub"
|
||||
echo "ASAN test content" > "$T/data/hello.txt"
|
||||
dd if=/dev/urandom bs=1024 count=200 of="$T/data/rand.bin" 2>/dev/null
|
||||
seq 1 20000 > "$T/data/sub/numbers.txt"
|
||||
yes "The quick brown fox. " | head -c 500000 > "$T/data/text.txt"
|
||||
|
||||
# Normal compress + extract (VaptVupt auto)
|
||||
./zupt_asan compress "$T/a1.zupt" "$T/data/" 2>&1
|
||||
./zupt_asan extract -o "$T/o1" "$T/a1.zupt" 2>&1
|
||||
|
||||
# Encrypted
|
||||
./zupt_asan compress -p "test123" "$T/a2.zupt" "$T/data/" 2>&1
|
||||
./zupt_asan extract -o "$T/o2" -p "test123" "$T/a2.zupt" 2>&1
|
||||
|
||||
# Explicit LZHP codec
|
||||
./zupt_asan compress --lzhp "$T/a3.zupt" "$T/data/" 2>&1
|
||||
./zupt_asan extract -o "$T/o3" "$T/a3.zupt" 2>&1
|
||||
|
||||
# Solid mode
|
||||
./zupt_asan compress --solid "$T/a4.zupt" "$T/data/" 2>&1
|
||||
./zupt_asan extract -o "$T/o4" "$T/a4.zupt" 2>&1
|
||||
|
||||
# Multi-threaded
|
||||
./zupt_asan compress -t 4 "$T/a5.zupt" "$T/data/" 2>&1
|
||||
./zupt_asan extract -o "$T/o5" "$T/a5.zupt" 2>&1
|
||||
|
||||
# PQ encryption
|
||||
./zupt_asan keygen -o "$T/priv.key" 2>&1
|
||||
./zupt_asan keygen --pub -o "$T/pub.key" -k "$T/priv.key" 2>&1
|
||||
./zupt_asan compress --pq "$T/pub.key" "$T/a6.zupt" "$T/data/" 2>&1
|
||||
./zupt_asan extract --pq "$T/priv.key" -o "$T/o6" "$T/a6.zupt" 2>&1
|
||||
|
||||
# Integrity test
|
||||
./zupt_asan test "$T/a1.zupt" 2>&1
|
||||
|
||||
echo "ASAN: all modes clean"
|
||||
rm -rf "$T"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Linux aarch64 — cross-compile + QEMU
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
build-linux-aarch64:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install aarch64 cross-compiler + QEMU
|
||||
run: |
|
||||
sudo apt-get update -q
|
||||
sudo apt-get install -y -q gcc-aarch64-linux-gnu qemu-user-static
|
||||
|
||||
- name: Build (aarch64, no Jasmin, no AVX2)
|
||||
run: |
|
||||
make CC=aarch64-linux-gnu-gcc \
|
||||
CFLAGS="-Wall -Wextra -O2 -std=c11 -Iinclude -Isrc -static" \
|
||||
LDFLAGS="-static" \
|
||||
LDLIBS="-lm -lpthread"
|
||||
|
||||
- name: Verify no Jasmin symbols (aarch64)
|
||||
run: |
|
||||
! nm zupt | grep -q "T zupt_aes256_blk" || \
|
||||
(echo "ERROR: Jasmin symbols should NOT be linked on aarch64" && exit 1)
|
||||
echo "Correct: no Jasmin symbols on aarch64"
|
||||
|
||||
- name: NIST/RFC test vectors (QEMU aarch64)
|
||||
run: |
|
||||
make test-vectors \
|
||||
CC=aarch64-linux-gnu-gcc \
|
||||
CFLAGS="-O2 -std=c11 -Iinclude -Isrc -static" \
|
||||
LDFLAGS="-static" \
|
||||
LDLIBS="-lm -lpthread"
|
||||
qemu-aarch64-static ./test_vectors
|
||||
|
||||
- name: VaptVupt unit tests (QEMU aarch64)
|
||||
run: |
|
||||
aarch64-linux-gnu-gcc -O2 -std=c11 -Iinclude -Isrc -static \
|
||||
tests/test_vaptvupt.c \
|
||||
src/vv_encoder.c src/vv_decoder.c src/vv_ans.c \
|
||||
src/vv_huffman.c src/vv_simd.c src/zupt_xxh.c src/zupt_cpuid.c \
|
||||
-lm -lpthread -o test_vaptvupt
|
||||
qemu-aarch64-static ./test_vaptvupt
|
||||
|
||||
- name: Smoke test (QEMU aarch64)
|
||||
run: |
|
||||
T=$(mktemp -d)
|
||||
echo "aarch64 test" > "$T/test.txt"
|
||||
seq 1 5000 >> "$T/test.txt"
|
||||
qemu-aarch64-static ./zupt compress "$T/a.zupt" "$T/test.txt" 2>&1
|
||||
qemu-aarch64-static ./zupt extract -o "$T/out" "$T/a.zupt" 2>&1
|
||||
EXTR=$(find "$T/out" -name test.txt -type f | head -1)
|
||||
diff -q "$T/test.txt" "$EXTR"
|
||||
echo "aarch64 roundtrip: OK"
|
||||
rm -rf "$T"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# macOS (Apple Silicon / Intel)
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
build-macos:
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Build
|
||||
run: make
|
||||
|
||||
- name: NIST/RFC test vectors
|
||||
run: make test-vectors && ./test_vectors
|
||||
|
||||
- name: VaptVupt unit tests
|
||||
run: make test-vv
|
||||
|
||||
- name: Regression tests
|
||||
run: bash tests/regression.sh
|
||||
|
||||
- name: Multi-threaded tests
|
||||
run: bash tests/test_threaded.sh
|
||||
|
||||
- name: Post-quantum tests
|
||||
run: bash tests/test_pq.sh ./zupt
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Windows (MSYS2 / MinGW-w64)
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
build-windows:
|
||||
runs-on: windows-latest
|
||||
defaults:
|
||||
run:
|
||||
shell: msys2 {0}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: MINGW64
|
||||
update: true
|
||||
install: >-
|
||||
mingw-w64-x86_64-gcc
|
||||
make
|
||||
diffutils
|
||||
|
||||
- name: Build
|
||||
run: make CC=gcc
|
||||
|
||||
- name: NIST/RFC test vectors
|
||||
run: make test-vectors && ./test_vectors
|
||||
|
||||
- name: VaptVupt unit tests
|
||||
run: make test-vv
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# PIE / Hardening build (distro packaging compliance)
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
build-hardened:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Build with PIE + full hardening
|
||||
run: |
|
||||
make CFLAGS="-Wall -Wextra -O2 -std=c11 -fPIE -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Iinclude -Isrc" \
|
||||
LDFLAGS="-pie -Wl,-z,relro,-z,now"
|
||||
|
||||
- name: Verify PIE binary
|
||||
run: |
|
||||
file ./zupt | grep -q "pie executable" || \
|
||||
(echo "ERROR: binary is not PIE" && exit 1)
|
||||
echo "PIE binary confirmed"
|
||||
|
||||
- name: Test install with DESTDIR
|
||||
run: |
|
||||
make install DESTDIR=/tmp/zupt_pkg
|
||||
test -f /tmp/zupt_pkg/usr/local/bin/zupt
|
||||
test -f /tmp/zupt_pkg/usr/local/share/man/man1/zupt.1.gz
|
||||
echo "Install layout OK"
|
||||
|
||||
- name: Full test suite on hardened build
|
||||
run: |
|
||||
make test-vectors && ./test_vectors
|
||||
make test-vv
|
||||
bash tests/regression.sh
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Release — create GitHub release with tarball on tag push
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
release:
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
needs: [build-linux, build-asan, build-linux-aarch64, build-macos, build-windows, build-hardened]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Build release tarball
|
||||
run: |
|
||||
VERSION="${GITHUB_REF_NAME#v}"
|
||||
make clean
|
||||
cd ..
|
||||
cp -a zupt "zupt-${VERSION}"
|
||||
tar czf "zupt-${VERSION}.tar.gz" "zupt-${VERSION}" \
|
||||
--exclude='*.o' --exclude='zupt-*/zupt' \
|
||||
--exclude='zupt_asan' --exclude='test_vectors' \
|
||||
--exclude='test_vaptvupt' --exclude='fuzz_*'
|
||||
mv "zupt-${VERSION}.tar.gz" zupt/
|
||||
cd zupt
|
||||
echo "TARBALL=zupt-${VERSION}.tar.gz" >> $GITHUB_ENV
|
||||
echo "VERSION=${VERSION}" >> $GITHUB_ENV
|
||||
|
||||
- name: Build binary
|
||||
run: make
|
||||
|
||||
- name: Run full test suite
|
||||
run: |
|
||||
make test-vectors && ./test_vectors
|
||||
make test-vv
|
||||
bash tests/regression.sh
|
||||
bash tests/test_threaded.sh
|
||||
bash tests/test_pq.sh ./zupt
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
name: "Zupt v${{ env.VERSION }}"
|
||||
body: |
|
||||
## Zupt v${{ env.VERSION }}
|
||||
|
||||
Backup compression with hardware-adaptive codec selection, AES-256 authenticated encryption, and post-quantum key encapsulation.
|
||||
|
||||
**Changes:** See [CHANGELOG.md](https://github.com/cristiancmoises/zupt/blob/master/CHANGELOG.md)
|
||||
|
||||
**Install:**
|
||||
```bash
|
||||
curl -fsSL https://short.securityops.co/zupt | bash
|
||||
```
|
||||
Or build from source:
|
||||
```bash
|
||||
tar xzf zupt-${{ env.VERSION }}.tar.gz && cd zupt-${{ env.VERSION }} && make && sudo make install
|
||||
```
|
||||
|
||||
**Test results:** 70/70 (11 VV + 13 NIST + 22 regression + 14 MT + 10 PQ)
|
||||
files: ${{ env.TARBALL }}
|
||||
generate_release_notes: true
|
||||
181
Makefile
181
Makefile
|
|
@ -1,19 +1,49 @@
|
|||
# Zupt v2.0.0 — Makefile with VaptVupt codec + Jasmin integration
|
||||
#
|
||||
# Multi-architecture: builds on x86_64, aarch64, armhf, ppc64le, s390x, riscv64.
|
||||
# Jasmin CT crypto: x86_64 only (C fallback on all other architectures).
|
||||
# AVX2 SIMD decode: x86_64 only (NEON on aarch64, scalar elsewhere).
|
||||
#
|
||||
# Packaging:
|
||||
# make Build with defaults
|
||||
# make V=1 Verbose build output
|
||||
# make install DESTDIR=/ Staged install for packaging
|
||||
# make CFLAGS="..." LDFLAGS="..." Override for distro policy (PIE, hardening)
|
||||
#
|
||||
# rpmlint / OBS compliance:
|
||||
# - LDFLAGS honored on link line (before objects for PIE)
|
||||
# - LDLIBS after objects (correct link order)
|
||||
# - DESTDIR support for staged installs
|
||||
# - Man page compressed and installed to $(MANDIR)/man1
|
||||
|
||||
CC ?= gcc
|
||||
CFLAGS ?= -Wall -Wextra -O2 -std=c11
|
||||
CFLAGS += -Iinclude -Isrc
|
||||
LDLIBS = -lm -lpthread
|
||||
LDFLAGS ?=
|
||||
LDLIBS ?= -lm -lpthread
|
||||
PREFIX ?= /usr/local
|
||||
BINDIR ?= $(PREFIX)/bin
|
||||
MANDIR ?= $(PREFIX)/share/man
|
||||
MAN1DIR ?= $(MANDIR)/man1
|
||||
GZIP ?= gzip
|
||||
GZIPFLAGS ?= -9 -n
|
||||
|
||||
# ─── Zupt core sources ───
|
||||
# --- Verbose build ---
|
||||
V ?= 0
|
||||
ifeq ($(V),1)
|
||||
Q =
|
||||
else
|
||||
Q = @
|
||||
endif
|
||||
|
||||
# --- Zupt core sources ---
|
||||
ZUPT_SOURCES = src/zupt_main.c src/zupt_format.c src/zupt_lz.c src/zupt_lzh.c \
|
||||
src/zupt_xxh.c src/zupt_sha256.c src/zupt_aes256.c src/zupt_crypto.c \
|
||||
src/zupt_predict.c src/zupt_parallel.c src/zupt_keccak.c \
|
||||
src/zupt_x25519.c src/zupt_mlkem.c src/zupt_cpuid.c src/zupt_mlock.c \
|
||||
src/zupt_filetype.c
|
||||
|
||||
# ─── VAPTVUPT: VaptVupt codec sources (Apache-2.0, integrated under MIT) ───
|
||||
# --- VAPTVUPT: VaptVupt codec sources (Apache-2.0, integrated under MIT) ---
|
||||
VV_SOURCES = src/vv_encoder.c src/vv_decoder.c src/vv_ans.c \
|
||||
src/vv_huffman.c src/vv_simd.c
|
||||
|
||||
|
|
@ -21,77 +51,127 @@ SOURCES = $(ZUPT_SOURCES) $(VV_SOURCES)
|
|||
|
||||
HEADERS = include/zupt.h include/zupt_keccak.h include/zupt_mlkem.h \
|
||||
include/zupt_x25519.h include/zupt_cpuid.h include/zupt_jasmin.h \
|
||||
include/zupt_acsl.h \
|
||||
include/vaptvupt.h include/vv_huffman.h include/vv_ans.h \
|
||||
src/zupt_thread.h src/zupt_parallel.h
|
||||
|
||||
TARGET = zupt
|
||||
MANPAGE = doc/zupt.1
|
||||
MANPAGE_GZ = $(TARGET).1.gz
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# ARCHITECTURE DETECTION
|
||||
#
|
||||
# Jasmin CT assembly: x86_64 only (pre-compiled .s files)
|
||||
# AVX2 SIMD decode: x86_64 only (-mavx2 on VV decode/encode/simd)
|
||||
# NEON SIMD decode: aarch64 (auto-detected by compiler, no extra flags)
|
||||
# Scalar fallback: all architectures
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
# ─── AVX2 detection: enable SIMD for VaptVupt on x86-64 ───
|
||||
ARCH := $(shell uname -m)
|
||||
|
||||
# --- AVX2: enable SIMD for VaptVupt on x86_64 ---
|
||||
ifeq ($(ARCH),x86_64)
|
||||
VV_SIMD_FLAGS = -mavx2
|
||||
else
|
||||
VV_SIMD_FLAGS =
|
||||
endif
|
||||
|
||||
# ─── Jasmin: use pre-compiled .s files if present ───
|
||||
JAZZ_S = jasmin/zupt_mac_verify.s jasmin/zupt_mlkem_select.s jasmin/zupt_aes_ctr.s jasmin/zupt_x25519_fe.s jasmin/zupt_aes_ctr4.s
|
||||
JAZZ_AVAILABLE := $(wildcard $(JAZZ_S))
|
||||
# --- Jasmin: enable only on x86_64 with pre-compiled .s files ---
|
||||
JAZZ_S = jasmin/zupt_mac_verify.s jasmin/zupt_mlkem_select.s \
|
||||
jasmin/zupt_aes_ctr.s jasmin/zupt_x25519_fe.s jasmin/zupt_aes_ctr4.s
|
||||
JAZZ_O =
|
||||
|
||||
ifeq ($(JAZZ_AVAILABLE),$(JAZZ_S))
|
||||
ifeq ($(ARCH),x86_64)
|
||||
JAZZ_AVAILABLE := $(wildcard $(JAZZ_S))
|
||||
ifeq ($(JAZZ_AVAILABLE),$(JAZZ_S))
|
||||
CFLAGS += -DZUPT_USE_JASMIN
|
||||
JAZZ_O = jasmin/zupt_mac_verify.o jasmin/zupt_mlkem_select.o jasmin/zupt_aes_ctr.o jasmin/zupt_x25519_fe.o jasmin/zupt_aes_ctr4.o
|
||||
$(info [jasmin] Verified assembly found — linking CT crypto)
|
||||
else
|
||||
JAZZ_O =
|
||||
JAZZ_O = jasmin/zupt_mac_verify.o jasmin/zupt_mlkem_select.o \
|
||||
jasmin/zupt_aes_ctr.o jasmin/zupt_x25519_fe.o jasmin/zupt_aes_ctr4.o
|
||||
$(info [jasmin] Enabled (x86_64) — linking CT crypto)
|
||||
else
|
||||
$(info [jasmin] Assembly not found — using C fallback)
|
||||
endif
|
||||
else
|
||||
$(info [jasmin] Disabled on $(ARCH) — using C fallback)
|
||||
endif
|
||||
|
||||
# ─── Object files for per-file CFLAGS (VV SIMD files need -mavx2) ───
|
||||
# --- Object files ---
|
||||
# VV SIMD files need -mavx2 on x86_64 (no-op on other arches)
|
||||
VV_SIMD_OBJS = src/vv_encoder.o src/vv_decoder.o src/vv_simd.o
|
||||
VV_PLAIN_OBJS = src/vv_ans.o src/vv_huffman.o
|
||||
ZUPT_OBJS = $(patsubst %.c,%.o,$(ZUPT_SOURCES))
|
||||
ALL_OBJS = $(ZUPT_OBJS) $(VV_SIMD_OBJS) $(VV_PLAIN_OBJS)
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# BUILD RULES
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
.PHONY: all clean install uninstall test test-all test-asan test-vectors test-vv fuzz-build help
|
||||
|
||||
all: $(TARGET)
|
||||
|
||||
# Jasmin pre-compiled assembly (x86_64 only)
|
||||
jasmin/%.o: jasmin/%.s
|
||||
$(CC) -c -o $@ $<
|
||||
$(Q)$(CC) $(CFLAGS) -c -o $@ $<
|
||||
|
||||
# VaptVupt SIMD files: compile with AVX2
|
||||
# VaptVupt SIMD files: compile with AVX2 on x86_64
|
||||
$(VV_SIMD_OBJS): src/%.o: src/%.c $(HEADERS)
|
||||
$(CC) $(CFLAGS) $(VV_SIMD_FLAGS) -c -o $@ $<
|
||||
$(Q)$(CC) $(CFLAGS) $(VV_SIMD_FLAGS) -c -o $@ $<
|
||||
|
||||
# VaptVupt non-SIMD files
|
||||
$(VV_PLAIN_OBJS): src/%.o: src/%.c $(HEADERS)
|
||||
$(CC) $(CFLAGS) -c -o $@ $<
|
||||
$(Q)$(CC) $(CFLAGS) -c -o $@ $<
|
||||
|
||||
# Zupt core files
|
||||
$(ZUPT_OBJS): src/%.o: src/%.c $(HEADERS)
|
||||
$(CC) $(CFLAGS) -c -o $@ $<
|
||||
$(Q)$(CC) $(CFLAGS) -c -o $@ $<
|
||||
|
||||
# Link: LDFLAGS before objects (for PIE), LDLIBS after (for -lm -lpthread)
|
||||
$(TARGET): $(ALL_OBJS) $(JAZZ_O)
|
||||
$(CC) $(CFLAGS) $(ALL_OBJS) $(JAZZ_O) $(LDLIBS) -o $(TARGET)
|
||||
@echo "Build complete: ./$(TARGET)"
|
||||
$(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(ALL_OBJS) $(JAZZ_O) -o $(TARGET) $(LDLIBS)
|
||||
@echo "Build complete: ./$(TARGET) [$(ARCH)]"
|
||||
|
||||
clean:
|
||||
rm -f $(TARGET) zupt_asan test_vectors test_vaptvupt fuzz_decompress fuzz_vv_decompress jasmin/*.o src/*.o
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# INSTALL / UNINSTALL
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
install: $(TARGET)
|
||||
@mkdir -p $(DESTDIR)$(BINDIR)
|
||||
install -m 755 $(TARGET) $(DESTDIR)$(BINDIR)/$(TARGET)
|
||||
$(Q)mkdir -p $(DESTDIR)$(BINDIR)
|
||||
$(Q)install -m 755 $(TARGET) $(DESTDIR)$(BINDIR)/$(TARGET)
|
||||
|
||||
$(Q)if [ -f "$(MANPAGE)" ]; then \
|
||||
mkdir -p $(DESTDIR)$(MAN1DIR); \
|
||||
$(GZIP) $(GZIPFLAGS) -c "$(MANPAGE)" > "$(DESTDIR)$(MAN1DIR)/$(MANPAGE_GZ)"; \
|
||||
chmod 0644 "$(DESTDIR)$(MAN1DIR)/$(MANPAGE_GZ)"; \
|
||||
echo "Installed: $(DESTDIR)$(MAN1DIR)/$(MANPAGE_GZ)"; \
|
||||
else \
|
||||
echo "Warning: man page not found: $(MANPAGE)"; \
|
||||
fi
|
||||
|
||||
@echo "Installed: $(DESTDIR)$(BINDIR)/$(TARGET)"
|
||||
|
||||
uninstall:
|
||||
rm -f $(DESTDIR)$(BINDIR)/$(TARGET)
|
||||
$(Q)rm -f $(DESTDIR)$(BINDIR)/$(TARGET)
|
||||
$(Q)rm -f $(DESTDIR)$(MAN1DIR)/$(MANPAGE_GZ)
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# CLEAN
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
clean:
|
||||
$(Q)rm -f $(TARGET) $(MANPAGE_GZ) zupt_asan test_vectors test_vaptvupt \
|
||||
fuzz_decompress fuzz_vv_decompress jasmin/*.o src/*.o
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# TEST TARGETS
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
test: $(TARGET)
|
||||
@sh tests/run_quick.sh
|
||||
$(Q)sh tests/run_quick.sh
|
||||
|
||||
test-all: $(TARGET) test-vectors test-vv
|
||||
@echo "═══════════════════════════════════════════════"
|
||||
@echo "==============================================="
|
||||
@sh tests/regression.sh 2>&1 | tail -3
|
||||
@echo ""
|
||||
@sh tests/test_threaded.sh 2>&1 | tail -3
|
||||
|
|
@ -101,46 +181,61 @@ test-all: $(TARGET) test-vectors test-vv
|
|||
@./test_vectors 2>&1 | tail -2
|
||||
@echo ""
|
||||
@./test_vaptvupt 2>&1 | tail -2
|
||||
@echo "═══════════════════════════════════════════════"
|
||||
@echo "==============================================="
|
||||
|
||||
test-vectors: tests/test_vectors.c $(HEADERS)
|
||||
$(CC) -O2 -std=c11 -Iinclude -Isrc tests/test_vectors.c \
|
||||
$(Q)$(CC) -O2 -std=c11 -Iinclude -Isrc $(LDFLAGS) tests/test_vectors.c \
|
||||
src/zupt_sha256.c src/zupt_crypto.c src/zupt_aes256.c src/zupt_xxh.c \
|
||||
src/zupt_keccak.c src/zupt_x25519.c src/zupt_mlkem.c src/zupt_cpuid.c \
|
||||
src/zupt_mlock.c \
|
||||
$(LDLIBS) -o test_vectors
|
||||
-o test_vectors $(LDLIBS)
|
||||
|
||||
# VAPTVUPT: VaptVupt codec unit tests
|
||||
test-vv: tests/test_vaptvupt.c $(HEADERS)
|
||||
$(CC) $(CFLAGS) $(VV_SIMD_FLAGS) tests/test_vaptvupt.c \
|
||||
$(Q)$(CC) $(CFLAGS) $(VV_SIMD_FLAGS) $(LDFLAGS) tests/test_vaptvupt.c \
|
||||
src/vv_encoder.c src/vv_decoder.c src/vv_ans.c src/vv_huffman.c \
|
||||
src/vv_simd.c src/zupt_xxh.c src/zupt_cpuid.c \
|
||||
$(LDLIBS) -o test_vaptvupt
|
||||
@./test_vaptvupt
|
||||
-o test_vaptvupt $(LDLIBS)
|
||||
$(Q)./test_vaptvupt
|
||||
|
||||
test-asan: $(SOURCES) $(HEADERS) $(JAZZ_O)
|
||||
$(CC) -Wall -Wextra -std=c11 -Iinclude -Isrc \
|
||||
$(Q)$(CC) -Wall -Wextra -std=c11 -Iinclude -Isrc \
|
||||
-fsanitize=address,undefined -g -O1 \
|
||||
$(VV_SIMD_FLAGS) \
|
||||
$(SOURCES) $(JAZZ_O) $(LDLIBS) -o zupt_asan
|
||||
$(VV_SIMD_FLAGS) $(LDFLAGS) \
|
||||
$(SOURCES) $(JAZZ_O) -o zupt_asan $(LDLIBS)
|
||||
@echo "ASAN build: ./zupt_asan"
|
||||
|
||||
# AFL++ fuzzing harnesses (requires afl-clang-fast)
|
||||
fuzz-build:
|
||||
@echo "Building AFL++ fuzzing harnesses..."
|
||||
afl-clang-fast -fsanitize=address,undefined -g -O1 -std=c11 \
|
||||
-Iinclude -Isrc $(VV_SIMD_FLAGS) \
|
||||
$(Q)afl-clang-fast -fsanitize=address,undefined -g -O1 -std=c11 \
|
||||
-Iinclude -Isrc $(VV_SIMD_FLAGS) $(LDFLAGS) \
|
||||
$(filter-out src/zupt_main.c,$(SOURCES)) tests/fuzz_decompress.c \
|
||||
$(LDLIBS) -o fuzz_decompress
|
||||
afl-clang-fast -fsanitize=address,undefined -g -O1 -std=c11 \
|
||||
-Iinclude -Isrc $(VV_SIMD_FLAGS) \
|
||||
-o fuzz_decompress $(LDLIBS)
|
||||
$(Q)afl-clang-fast -fsanitize=address,undefined -g -O1 -std=c11 \
|
||||
-Iinclude -Isrc $(VV_SIMD_FLAGS) $(LDFLAGS) \
|
||||
tests/fuzz_vv_decompress.c \
|
||||
src/vv_encoder.c src/vv_decoder.c src/vv_ans.c src/vv_huffman.c \
|
||||
src/vv_simd.c src/zupt_xxh.c src/zupt_cpuid.c \
|
||||
$(LDLIBS) -o fuzz_vv_decompress
|
||||
-o fuzz_vv_decompress $(LDLIBS)
|
||||
@echo "Fuzz harnesses built. Run:"
|
||||
@echo " afl-fuzz -i corpus -o findings -- ./fuzz_decompress"
|
||||
@echo " afl-fuzz -i corpus_vv -o findings_vv -- ./fuzz_vv_decompress"
|
||||
|
||||
help:
|
||||
@echo "make / make test / make install / make test-all / make test-asan / make test-vv / make fuzz-build / make clean"
|
||||
@echo "Zupt v2.0.0 build targets:"
|
||||
@echo " make Build zupt binary"
|
||||
@echo " make V=1 Build with verbose output"
|
||||
@echo " make test Quick test"
|
||||
@echo " make test-all Full test suite (regression + threaded + PQ + vectors + VV)"
|
||||
@echo " make test-vv VaptVupt codec unit tests"
|
||||
@echo " make test-asan Build with AddressSanitizer"
|
||||
@echo " make fuzz-build Build AFL++ fuzzing harnesses"
|
||||
@echo " make install Install to $(PREFIX)"
|
||||
@echo " make uninstall Remove from $(PREFIX)"
|
||||
@echo " make clean Remove build artifacts"
|
||||
@echo ""
|
||||
@echo "Architecture: $(ARCH)"
|
||||
@echo " x86_64: Jasmin CT crypto + AVX2 SIMD decode"
|
||||
@echo " aarch64: C crypto fallback + NEON SIMD decode"
|
||||
@echo " other: C crypto fallback + scalar decode"
|
||||
|
|
|
|||
|
|
@ -7,8 +7,6 @@
|
|||
* VaptVupt Codec — Next-generation lossless compression
|
||||
* Public API and data structures
|
||||
*
|
||||
* SPDX-License-Identifier: GPL-3.0-or-later
|
||||
* Copyright 2026 Cristian.
|
||||
* Zero dependencies. Pure C11.
|
||||
*/
|
||||
#ifndef VAPTVUPT_H
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ vva_error_t vva_decode_ctx(const uint8_t *src, size_t src_len,
|
|||
*/
|
||||
|
||||
#define VVA_ML_CODES 36 /* Match length code count */
|
||||
#define VVA_OF_CODES 24 /* Offset code count */
|
||||
#define VVA_OF_CODES 27 /* Offset code count: 3 rep + 24 explicit */
|
||||
|
||||
vva_error_t vva_encode_sequences(const uint8_t *tokens, size_t tok_len,
|
||||
uint8_t *dst, size_t dst_cap, size_t *dst_len,
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@
|
|||
#define ZUPT_CODEC_ZUPT_LZH 0x0009 /* LZ77 + Huffman */
|
||||
#define ZUPT_CODEC_ZUPT_LZHP 0x000A /* LZ77 + Huffman + Byte Prediction (default) */
|
||||
#define ZUPT_CODEC_VAPTVUPT 0x0010 /* VAPTVUPT: VaptVupt LZ + ANS entropy codec */
|
||||
#define ZUPT_CODEC_AUTO 0xFFFF /* Auto-detect: VaptVupt if AVX2, else LZHP */
|
||||
|
||||
/* Crypto */
|
||||
#define ZUPT_SALT_SIZE 32
|
||||
|
|
@ -326,4 +327,10 @@ const char *zupt_codec_name(uint16_t id);
|
|||
void zupt_default_options(zupt_options_t *o);
|
||||
void zupt_format_size(uint64_t bytes, char *buf, size_t cap);
|
||||
|
||||
/* Resolve ZUPT_CODEC_AUTO to a concrete codec based on hardware.
|
||||
* On x86_64 with AVX2: VaptVupt (fast ANS+SIMD decode).
|
||||
* On all other arches: Zupt-LZHP (no SIMD dependency).
|
||||
* Decompression of ALL codecs works on ALL architectures. */
|
||||
uint16_t zupt_resolve_auto_codec(void);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -9,13 +9,15 @@
|
|||
|
||||
typedef struct {
|
||||
int has_aesni; /* CPUID.01H:ECX[25] — AES-NI instructions */
|
||||
int has_avx; /* AVX (VEX-encoded SSE) — requires CPUID + OS XSAVE */
|
||||
int has_pclmul; /* CPUID.01H:ECX[1] — CLMUL (carry-less multiply) */
|
||||
int has_avx2; /* CPUID.07H:EBX[5] — AVX2 (256-bit SIMD) */
|
||||
int has_sse41; /* CPUID.01H:ECX[19] — SSE4.1 */
|
||||
} zupt_cpu_features_t;
|
||||
|
||||
/*@ assigns f->has_aesni, f->has_pclmul, f->has_avx2, f->has_sse41;
|
||||
/*@ assigns f->has_aesni, f->has_avx, f->has_pclmul, f->has_avx2, f->has_sse41;
|
||||
@ ensures f->has_aesni == 0 || f->has_aesni == 1;
|
||||
@ ensures f->has_avx == 0 || f->has_avx == 1;
|
||||
@ ensures f->has_pclmul == 0 || f->has_pclmul == 1;
|
||||
@ ensures f->has_avx2 == 0 || f->has_avx2 == 1;
|
||||
@ ensures f->has_sse41 == 0 || f->has_sse41 == 1;
|
||||
|
|
|
|||
112
src/vv_ans.c
112
src/vv_ans.c
|
|
@ -6,6 +6,7 @@
|
|||
#if !defined(_DEFAULT_SOURCE) && !defined(_GNU_SOURCE)
|
||||
#define _DEFAULT_SOURCE 1
|
||||
#endif
|
||||
|
||||
/*
|
||||
* VaptVupt — tANS v2 (sparse header + 4-way interleaved decode)
|
||||
*
|
||||
|
|
@ -1170,7 +1171,11 @@ static const uint8_t ml_extra[VVA_ML_CODES] = {
|
|||
12,13,14,15
|
||||
};
|
||||
|
||||
/* Rep-match codes: 0=rep[0], 1=rep[1], 2=rep[2], 3+=explicit offset.
|
||||
* Explicit offset code c (c≥3): offset in [2^(c-3), 2^(c-2)), (c-3) extra bits.
|
||||
* This is how zstd encodes repeated offsets — saves 10-15 bits per rep-match. */
|
||||
static const uint8_t of_extra[VVA_OF_CODES] = {
|
||||
0,0,0, /* rep codes: 0 extra bits */
|
||||
0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23
|
||||
};
|
||||
|
||||
|
|
@ -1192,21 +1197,24 @@ static uint32_t ml_decode(uint8_t code, uint32_t extra) {
|
|||
return ml_base[code] + extra;
|
||||
}
|
||||
|
||||
/* Encode offset → (code, extra_value, extra_bits) */
|
||||
/* Encode explicit offset → (code, extra_value, extra_bits).
|
||||
* Returns code in range [3..26]. Caller handles rep-match codes 0-2. */
|
||||
static void of_encode(uint32_t offset, uint8_t *code, uint32_t *extra, int *nbits) {
|
||||
if (offset == 0) { *code = 0; *extra = 0; *nbits = 0; return; }
|
||||
if (offset == 0) { *code = 3; *extra = 0; *nbits = 0; return; }
|
||||
int c = 0;
|
||||
uint32_t v = offset;
|
||||
while (v > 1) { v >>= 1; c++; }
|
||||
if (c >= VVA_OF_CODES) c = VVA_OF_CODES - 1;
|
||||
*code = (uint8_t)c;
|
||||
if (c >= 24) c = 23; /* clamp to 24 explicit codes */
|
||||
*code = (uint8_t)(c + 3); /* shift by 3 for rep codes */
|
||||
*extra = offset - (1u << c);
|
||||
*nbits = of_extra[c];
|
||||
*nbits = of_extra[c + 3];
|
||||
}
|
||||
|
||||
/* Decode offset code → offset */
|
||||
/* Decode offset code → offset. Codes 0-2 are rep-match (caller resolves).
|
||||
* Codes 3-26 are explicit offsets. */
|
||||
static uint32_t of_decode(uint8_t code, uint32_t extra) {
|
||||
return (1u << code) + extra;
|
||||
if (code < 3) return 0; /* rep-match — caller must handle */
|
||||
return (1u << (code - 3)) + extra;
|
||||
}
|
||||
|
||||
/* Write a varint to a buffer, return bytes written */
|
||||
|
|
@ -1349,21 +1357,61 @@ vva_error_t vva_encode_sequences(const uint8_t *tokens, size_t tok_len,
|
|||
}
|
||||
}
|
||||
|
||||
/* ─── Count ML and OF code frequencies ─── */
|
||||
/* ─── Count ML and OF code frequencies with rep-match tracking ─── */
|
||||
uint32_t freq_ml[VVA_ML_CODES], freq_of[VVA_OF_CODES];
|
||||
memset(freq_ml, 0, sizeof(freq_ml));
|
||||
memset(freq_of, 0, sizeof(freq_of));
|
||||
|
||||
/* Precompute OF codes with rep-match detection (forward pass).
|
||||
* Store in per-sequence arrays so the backward ANS pass can use them. */
|
||||
uint8_t *seq_of_code = NULL;
|
||||
uint32_t *seq_of_extra = NULL;
|
||||
int *seq_of_nbits = NULL;
|
||||
seq_of_code = (uint8_t *)malloc(nseq * sizeof(uint8_t));
|
||||
seq_of_extra = (uint32_t *)malloc(nseq * sizeof(uint32_t));
|
||||
seq_of_nbits = (int *)malloc(nseq * sizeof(int));
|
||||
if (!seq_of_code || !seq_of_extra || !seq_of_nbits) {
|
||||
free(seq_of_code); free(seq_of_extra); free(seq_of_nbits);
|
||||
free(seqs); free(lit_buf); free(lit_enc);
|
||||
return VVA_ERR_NOMEM;
|
||||
}
|
||||
|
||||
size_t match_count = 0;
|
||||
uint32_t enc_rep[3] = {0, 0, 0}; /* Rep-match tracking during forward pass */
|
||||
for (size_t i = 0; i < nseq; i++) {
|
||||
if (seqs[i].matchlen > 0) {
|
||||
uint8_t mc; uint32_t mx; int mn;
|
||||
ml_encode(seqs[i].matchlen, &mc, &mx, &mn);
|
||||
freq_ml[mc]++;
|
||||
|
||||
/* Check rep-match before explicit encoding */
|
||||
uint32_t off = seqs[i].offset;
|
||||
uint8_t oc; uint32_t ox; int on;
|
||||
of_encode(seqs[i].offset, &oc, &ox, &on);
|
||||
if (off == enc_rep[0] && off != 0) {
|
||||
oc = 0; ox = 0; on = 0; /* rep[0] */
|
||||
} else if (off == enc_rep[1] && off != 0) {
|
||||
oc = 1; ox = 0; on = 0; /* rep[1] */
|
||||
} else if (off == enc_rep[2] && off != 0) {
|
||||
oc = 2; ox = 0; on = 0; /* rep[2] */
|
||||
} else {
|
||||
of_encode(off, &oc, &ox, &on); /* explicit: codes 3-26 */
|
||||
}
|
||||
seq_of_code[i] = oc;
|
||||
seq_of_extra[i] = ox;
|
||||
seq_of_nbits[i] = on;
|
||||
freq_of[oc]++;
|
||||
|
||||
/* Update rep array (same logic as LZ engine) */
|
||||
if (off != enc_rep[0] && off != 0) {
|
||||
enc_rep[2] = enc_rep[1];
|
||||
enc_rep[1] = enc_rep[0];
|
||||
enc_rep[0] = off;
|
||||
}
|
||||
match_count++;
|
||||
} else {
|
||||
seq_of_code[i] = 0;
|
||||
seq_of_extra[i] = 0;
|
||||
seq_of_nbits[i] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1440,11 +1488,15 @@ vva_error_t vva_encode_sequences(const uint8_t *tokens, size_t tok_len,
|
|||
for (size_t ii = nseq; ii > 0; ii--) {
|
||||
if (seqs[ii - 1].matchlen == 0) continue;
|
||||
|
||||
uint8_t mc, oc;
|
||||
uint32_t mx, ox;
|
||||
int mn, on;
|
||||
uint8_t mc;
|
||||
uint32_t mx;
|
||||
int mn;
|
||||
ml_encode(seqs[ii - 1].matchlen, &mc, &mx, &mn);
|
||||
of_encode(seqs[ii - 1].offset, &oc, &ox, &on);
|
||||
|
||||
/* Use precomputed OF code from forward pass (rep-match aware) */
|
||||
uint8_t oc = seq_of_code[ii - 1];
|
||||
uint32_t ox = seq_of_extra[ii - 1];
|
||||
int on = seq_of_nbits[ii - 1];
|
||||
|
||||
/* Encode in this order (reversed): ml_code, ml_extra, of_code, of_extra
|
||||
* Decoder reads: of_extra, of_code, ml_extra, ml_code */
|
||||
|
|
@ -1513,7 +1565,15 @@ vva_error_t vva_encode_sequences(const uint8_t *tokens, size_t tok_len,
|
|||
}
|
||||
|
||||
/* ─── Encode litlen varints ─── */
|
||||
litlen_buf = (uint8_t *)malloc(nseq * 5 + 1);
|
||||
/* Each litlen varint uses ceil(litlen/255)+1 bytes. Worst case for
|
||||
* nseq sequences each with large litlen: compute exact bound. */
|
||||
{
|
||||
size_t litlen_cap = nseq; /* at least 1 byte per sequence */
|
||||
for (size_t i = 0; i < nseq; i++)
|
||||
litlen_cap += seqs[i].litlen / 255;
|
||||
litlen_cap += 16; /* safety margin */
|
||||
litlen_buf = (uint8_t *)malloc(litlen_cap);
|
||||
}
|
||||
if (!litlen_buf) goto seq_fail;
|
||||
{
|
||||
size_t pos = 0;
|
||||
|
|
@ -1572,11 +1632,13 @@ vva_error_t vva_encode_sequences(const uint8_t *tokens, size_t tok_len,
|
|||
}
|
||||
|
||||
free(seqs); free(lit_buf); free(lit_enc);
|
||||
free(seq_of_code); free(seq_of_extra); free(seq_of_nbits);
|
||||
free(ml_hdr_buf); free(of_hdr_buf); free(seq_bs); free(litlen_buf);
|
||||
return VVA_OK;
|
||||
|
||||
seq_fail:
|
||||
free(seqs); free(lit_buf); free(lit_enc);
|
||||
free(seq_of_code); free(seq_of_extra); free(seq_of_nbits);
|
||||
free(ml_hdr_buf); free(of_hdr_buf); free(seq_bs); free(litlen_buf);
|
||||
return VVA_ERR_OVERFLOW;
|
||||
}
|
||||
|
|
@ -1691,15 +1753,11 @@ vva_error_t vva_decode_sequences(const uint8_t *src, size_t src_len,
|
|||
uint8_t *op_end = dst + dst_cap;
|
||||
size_t lit_pos = 0;
|
||||
size_t matches_decoded = 0;
|
||||
uint32_t dec_rep[3] = {0, 0, 0}; /* Rep-match offset tracking */
|
||||
|
||||
/* We don't know exact nseq, but we have match_count matches + possibly
|
||||
* one final literal-only sequence. Decode until all literals consumed
|
||||
* and all matches decoded. */
|
||||
while (lit_pos < total_lits || matches_decoded < match_count) {
|
||||
/* Read litlen from varint stream */
|
||||
size_t litlen = seq_read_varint(&ll_p, end);
|
||||
|
||||
/* Copy literals from decoded literal buffer */
|
||||
if (lit_pos + litlen > total_lits) { free(dec_ml); free(dec_of); free(lit_buf); return VVA_ERR_CORRUPT; }
|
||||
if (op + litlen > op_end) { free(dec_ml); free(dec_of); free(lit_buf); return VVA_ERR_OVERFLOW; }
|
||||
if (litlen > 0) {
|
||||
|
|
@ -1708,23 +1766,31 @@ vva_error_t vva_decode_sequences(const uint8_t *src, size_t src_len,
|
|||
lit_pos += litlen;
|
||||
}
|
||||
|
||||
/* If all matches decoded, this was the last literal-only sequence */
|
||||
if (matches_decoded >= match_count) break;
|
||||
|
||||
/* Decode offset */
|
||||
/* Decode OF code */
|
||||
if (r.n < ANS_LOG) br_fill(&r);
|
||||
if (state_of >= (uint32_t)ANS_L) { free(dec_ml); free(dec_of); free(lit_buf); return VVA_ERR_CORRUPT; }
|
||||
vva_dec_entry_t eof = dec_of[state_of];
|
||||
uint32_t of_bits = br_read(&r, eof.nbits);
|
||||
state_of = (uint32_t)eof.baseline + of_bits;
|
||||
|
||||
/* Read offset extra bits */
|
||||
/* Resolve offset: codes 0-2 = rep-match, 3+ = explicit */
|
||||
uint8_t of_code = eof.symbol;
|
||||
uint32_t offset;
|
||||
if (of_code < 3) {
|
||||
offset = dec_rep[of_code];
|
||||
} else {
|
||||
uint32_t of_extra_val = 0;
|
||||
if (of_code < VVA_OF_CODES && of_extra[of_code] > 0) {
|
||||
of_extra_val = br_read(&r, of_extra[of_code]);
|
||||
}
|
||||
uint32_t offset = of_decode(of_code, of_extra_val);
|
||||
offset = of_decode(of_code, of_extra_val);
|
||||
}
|
||||
/* Update rep offsets */
|
||||
if (offset != 0 && offset != dec_rep[0]) {
|
||||
dec_rep[2] = dec_rep[1]; dec_rep[1] = dec_rep[0]; dec_rep[0] = offset;
|
||||
}
|
||||
|
||||
/* Decode match length */
|
||||
if (r.n < ANS_LOG) br_fill(&r);
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#if !defined(_DEFAULT_SOURCE) && !defined(_GNU_SOURCE)
|
||||
#define _DEFAULT_SOURCE 1
|
||||
#endif
|
||||
|
||||
/*
|
||||
* VaptVupt — Decoder v2 (Sprint 1)
|
||||
*
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#if !defined(_DEFAULT_SOURCE) && !defined(_GNU_SOURCE)
|
||||
#define _DEFAULT_SOURCE 1
|
||||
#endif
|
||||
|
||||
/*
|
||||
* VaptVupt — Encoder v2 (Sprint 1)
|
||||
*
|
||||
|
|
@ -442,11 +443,8 @@ int64_t vv_compress(const uint8_t *src, size_t src_len,
|
|||
if (wlog == 0) {
|
||||
switch (opts->mode) {
|
||||
case VV_MODE_ULTRA_FAST: wlog = 16; break;
|
||||
case VV_MODE_BALANCED: wlog = 16; break;
|
||||
case VV_MODE_EXTREME: wlog = 16; break;
|
||||
/* TRADEOFF: wlog=16 default avoids 3-byte offset overhead on small data.
|
||||
* Users can set opts.window_log=20 (1MB) or 22 (4MB) for large files
|
||||
* with long-range patterns. Zupt sets wlog=20 for backup chunks >1MB. */
|
||||
case VV_MODE_BALANCED: wlog = 16; break; /* may be overridden below */
|
||||
case VV_MODE_EXTREME: wlog = 16; break; /* may be overridden below */
|
||||
}
|
||||
}
|
||||
switch (opts->mode) {
|
||||
|
|
@ -456,6 +454,41 @@ int64_t vv_compress(const uint8_t *src, size_t src_len,
|
|||
default: depth = 48;
|
||||
}
|
||||
|
||||
/* ─── ADAPTIVE WINDOW (Item 2): trial-compress first block at wlog=16
|
||||
* and wlog=20. If wlog=20 produces ≥3% smaller output, use it.
|
||||
* Only for balanced/extreme with auto wlog (opts->window_log == 0).
|
||||
* Cost: one extra compression of the first block (~10ms for 1MB).
|
||||
* TRADEOFF: encode speed vs automatic ratio optimization.
|
||||
* Zupt benefits because backup data characteristics are unknown. ─── */
|
||||
if (opts->window_log == 0 && opts->mode >= VV_MODE_BALANCED && src_len > 65536) {
|
||||
size_t trial_len = src_len;
|
||||
if (trial_len > VV_MAX_BLOCK_SIZE) trial_len = VV_MAX_BLOCK_SIZE;
|
||||
|
||||
size_t trial_cap = trial_len + trial_len / 255 + 1024;
|
||||
uint8_t *trial_buf = (uint8_t *)malloc(trial_cap);
|
||||
if (trial_buf) {
|
||||
/* Trial at wlog=16 */
|
||||
matcher_t m16;
|
||||
matcher_init(&m16, 16, depth);
|
||||
size_t sz16 = compress_block(src, trial_len, trial_buf, trial_cap, &m16, opts->mode);
|
||||
matcher_free(&m16);
|
||||
|
||||
/* Trial at wlog=20 */
|
||||
matcher_t m20;
|
||||
matcher_init(&m20, 20, depth);
|
||||
size_t sz20 = compress_block(src, trial_len, trial_buf, trial_cap, &m20, opts->mode);
|
||||
matcher_free(&m20);
|
||||
|
||||
free(trial_buf);
|
||||
|
||||
/* Pick winner: wlog=20 must save ≥3% to justify 3-byte offsets */
|
||||
if (sz20 > 0 && sz16 > 0 && sz20 < (sz16 * 97 / 100)) {
|
||||
wlog = 20;
|
||||
}
|
||||
/* Otherwise stay at wlog=16 (no regression on short-offset data) */
|
||||
}
|
||||
}
|
||||
|
||||
/* Frame header */
|
||||
uint8_t *op = dst;
|
||||
vv_frame_header_t fh;
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#if !defined(_DEFAULT_SOURCE) && !defined(_GNU_SOURCE)
|
||||
#define _DEFAULT_SOURCE 1
|
||||
#endif
|
||||
|
||||
/*
|
||||
* VaptVupt — Canonical Huffman Codec Implementation
|
||||
*
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#if !defined(_DEFAULT_SOURCE) && !defined(_GNU_SOURCE)
|
||||
#define _DEFAULT_SOURCE 1
|
||||
#endif
|
||||
|
||||
/*
|
||||
* VaptVupt — SIMD-accelerated copy routines
|
||||
*
|
||||
|
|
@ -38,8 +39,8 @@ static void copy_match_scalar(uint8_t *dst, uint32_t offset, size_t length) {
|
|||
dst += 16; src += 16; length -= 16;
|
||||
}
|
||||
if (length > 0) memcpy(dst, src, length);
|
||||
} else if (offset >= 4) {
|
||||
/* Moderate overlap: 8-byte copy with re-read */
|
||||
} else if (offset >= 8) {
|
||||
/* Moderate overlap (8-15): 8-byte copy is safe since offset >= stride */
|
||||
while (length >= 8) {
|
||||
uint64_t v;
|
||||
memcpy(&v, src, 8);
|
||||
|
|
@ -48,7 +49,7 @@ static void copy_match_scalar(uint8_t *dst, uint32_t offset, size_t length) {
|
|||
}
|
||||
while (length-- > 0) *dst++ = *src++;
|
||||
} else {
|
||||
/* Very short overlap (1-3): byte-by-byte */
|
||||
/* Short overlap (1-7): byte-by-byte to handle pattern repeat correctly */
|
||||
for (size_t i = 0; i < length; i++) dst[i] = src[i];
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
#include <string.h>
|
||||
|
||||
/* Global instance */
|
||||
zupt_cpu_features_t zupt_cpu = {0, 0, 0, 0};
|
||||
zupt_cpu_features_t zupt_cpu = {0, 0, 0, 0, 0};
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════
|
||||
* CPUID intrinsics — platform-specific
|
||||
|
|
@ -30,6 +30,9 @@ zupt_cpu_features_t zupt_cpu = {0, 0, 0, 0};
|
|||
__cpuidex(regs, leaf, subleaf);
|
||||
*eax = regs[0]; *ebx = regs[1]; *ecx = regs[2]; *edx = regs[3];
|
||||
}
|
||||
static uint64_t zupt_xgetbv(uint32_t idx) {
|
||||
return _xgetbv(idx);
|
||||
}
|
||||
#elif defined(__GNUC__) || defined(__clang__)
|
||||
#include <cpuid.h>
|
||||
static void zupt_cpuid(int leaf, int subleaf, int *eax, int *ebx, int *ecx, int *edx) {
|
||||
|
|
@ -37,6 +40,11 @@ zupt_cpu_features_t zupt_cpu = {0, 0, 0, 0};
|
|||
__cpuid_count((unsigned int)leaf, (unsigned int)subleaf, a, b, c, d);
|
||||
*eax = (int)a; *ebx = (int)b; *ecx = (int)c; *edx = (int)d;
|
||||
}
|
||||
static uint64_t zupt_xgetbv(uint32_t idx) {
|
||||
uint32_t lo, hi;
|
||||
__asm__ __volatile__ ("xgetbv" : "=a"(lo), "=d"(hi) : "c"(idx));
|
||||
return ((uint64_t)hi << 32) | lo;
|
||||
}
|
||||
#else
|
||||
/* Inline assembly fallback */
|
||||
static void zupt_cpuid(int leaf, int subleaf, int *eax, int *ebx, int *ecx, int *edx) {
|
||||
|
|
@ -46,6 +54,11 @@ zupt_cpu_features_t zupt_cpu = {0, 0, 0, 0};
|
|||
: "a"(leaf), "c"(subleaf)
|
||||
);
|
||||
}
|
||||
static uint64_t zupt_xgetbv(uint32_t idx) {
|
||||
uint32_t lo, hi;
|
||||
__asm__ __volatile__ ("xgetbv" : "=a"(lo), "=d"(hi) : "c"(idx));
|
||||
return ((uint64_t)hi << 32) | lo;
|
||||
}
|
||||
#endif
|
||||
|
||||
void zupt_detect_cpu(zupt_cpu_features_t *f) {
|
||||
|
|
@ -62,11 +75,27 @@ void zupt_detect_cpu(zupt_cpu_features_t *f) {
|
|||
f->has_aesni = (ecx >> 25) & 1; /* ECX bit 25 */
|
||||
f->has_pclmul = (ecx >> 1) & 1; /* ECX bit 1 */
|
||||
f->has_sse41 = (ecx >> 19) & 1; /* ECX bit 19 */
|
||||
|
||||
/* AVX detection: CPU must support AVX (ECX[28]) AND the OS must
|
||||
* have enabled XSAVE/XRSTOR for YMM state (OSXSAVE, ECX[27]).
|
||||
* If OSXSAVE is set, check XCR0 bits 1+2 (SSE+AVX state).
|
||||
* Without this check, VEX-encoded instructions (vaesenc, vmovdqu,
|
||||
* vpxor, etc.) will SIGILL even if the CPU supports them. */
|
||||
int has_avx_cpu = (ecx >> 28) & 1;
|
||||
int has_osxsave = (ecx >> 27) & 1;
|
||||
if (has_avx_cpu && has_osxsave) {
|
||||
uint64_t xcr0 = zupt_xgetbv(0);
|
||||
/* Bits 1 (SSE/XMM) and 2 (AVX/YMM) must both be set */
|
||||
if ((xcr0 & 0x6) == 0x6)
|
||||
f->has_avx = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (max_leaf >= 7) {
|
||||
zupt_cpuid(7, 0, &eax, &ebx, &ecx, &edx);
|
||||
f->has_avx2 = (ebx >> 5) & 1; /* EBX bit 5 */
|
||||
/* AVX2 also requires AVX (OS XSAVE) to be usable */
|
||||
if (f->has_avx && ((ebx >> 5) & 1))
|
||||
f->has_avx2 = 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -74,7 +103,7 @@ void zupt_detect_cpu(zupt_cpu_features_t *f) {
|
|||
|
||||
void zupt_detect_cpu(zupt_cpu_features_t *f) {
|
||||
memset(f, 0, sizeof(*f));
|
||||
/* No AES-NI on ARM/RISC-V/etc — use table fallback */
|
||||
/* No AES-NI/AVX on ARM/RISC-V/etc — use table fallback */
|
||||
}
|
||||
|
||||
#endif /* ZUPT_HAS_CPUID */
|
||||
|
|
|
|||
|
|
@ -189,9 +189,11 @@ void zupt_aes256_ctr(const uint8_t key[32], const uint8_t nonce[16],
|
|||
|
||||
#ifdef ZUPT_USE_JASMIN
|
||||
/* JASMIN-VERIFIED: AES-NI path — constant-time, no T-table leakage.
|
||||
* Requires AES-NI support (detected via CPUID at startup).
|
||||
* Uses 4-block pipeline for bulk data, single-block for tail. */
|
||||
if (zupt_cpu.has_aesni) {
|
||||
* The Jasmin-generated assembly uses VEX-encoded instructions (vaesenc,
|
||||
* vmovdqu, vpxor, etc.) which require BOTH AES-NI AND AVX support.
|
||||
* Checking only has_aesni would SIGILL on CPUs with AES-NI but no AVX,
|
||||
* or where the OS hasn't enabled XSAVE for YMM state. */
|
||||
if (zupt_cpu.has_aesni && zupt_cpu.has_avx) {
|
||||
size_t full_blocks = len / 16;
|
||||
size_t tail_bytes = len % 16;
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include "zupt.h"
|
||||
#include "zupt_cpuid.h" /* zupt_cpu for AUTO codec detection */
|
||||
#include "zupt_parallel.h"
|
||||
#include "vaptvupt.h" /* VAPTVUPT: VaptVupt codec integration */
|
||||
#include <stdio.h>
|
||||
|
|
@ -51,6 +52,7 @@ const char *zupt_codec_name(uint16_t id) {
|
|||
case ZUPT_CODEC_ZUPT_LZH: return "Zupt-LZH";
|
||||
case ZUPT_CODEC_ZUPT_LZHP: return "Zupt-LZHP";
|
||||
case ZUPT_CODEC_VAPTVUPT: return "VaptVupt"; /* VAPTVUPT */
|
||||
case ZUPT_CODEC_AUTO: return "Auto";
|
||||
default: return "Unknown";
|
||||
}
|
||||
}
|
||||
|
|
@ -58,12 +60,41 @@ void zupt_default_options(zupt_options_t *o) {
|
|||
memset(o, 0, sizeof(*o));
|
||||
o->level = 7;
|
||||
o->block_size = 0;
|
||||
o->codec_id = ZUPT_CODEC_VAPTVUPT; /* VAPTVUPT: default codec v2.0.0 */
|
||||
o->codec_id = ZUPT_CODEC_AUTO; /* Auto-detect: VaptVupt if AVX2, else LZHP */
|
||||
/* Init keyring canaries */
|
||||
o->keyring.canary_head = ZUPT_CANARY;
|
||||
o->keyring.canary_tail = ZUPT_CANARY;
|
||||
}
|
||||
|
||||
/* Resolve ZUPT_CODEC_AUTO to a concrete codec.
|
||||
* VaptVupt decode works on ALL architectures (scalar fallback), but the
|
||||
* AVX2 SIMD decode path gives ~3× throughput. On non-AVX2 hardware,
|
||||
* Zupt-LZHP is a better default since its simpler decoder doesn't
|
||||
* benefit from SIMD as much.
|
||||
*
|
||||
* Detection order:
|
||||
* 1. Compile-time: __x86_64__ + __AVX2__ → VaptVupt (compiled with -mavx2)
|
||||
* 2. Runtime: zupt_cpu.has_avx2 → VaptVupt (for x86_64 without -mavx2)
|
||||
* 3. Compile-time: __aarch64__ + __ARM_NEON → VaptVupt (NEON decode)
|
||||
* 4. Fallback: Zupt-LZHP (works everywhere)
|
||||
*/
|
||||
uint16_t zupt_resolve_auto_codec(void) {
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
/* x86_64: check AVX2 at compile time (via -mavx2) or runtime (cpuid) */
|
||||
#if defined(__AVX2__)
|
||||
return ZUPT_CODEC_VAPTVUPT; /* Compiled with -mavx2: inline SIMD decode */
|
||||
#else
|
||||
if (zupt_cpu.has_avx2)
|
||||
return ZUPT_CODEC_VAPTVUPT; /* Runtime AVX2: vv_simd.c dispatch */
|
||||
return ZUPT_CODEC_ZUPT_LZHP; /* No AVX2: use LZHP */
|
||||
#endif
|
||||
#elif defined(__aarch64__) && defined(__ARM_NEON)
|
||||
return ZUPT_CODEC_VAPTVUPT; /* NEON SIMD decode available */
|
||||
#else
|
||||
return ZUPT_CODEC_ZUPT_LZHP; /* Scalar only: LZHP is a better default */
|
||||
#endif
|
||||
}
|
||||
|
||||
static uint32_t auto_block_size(int level) {
|
||||
if (level <= 2) return 131072;
|
||||
if (level <= 4) return 131072;
|
||||
|
|
@ -276,6 +307,10 @@ zupt_error_t zupt_compress_files(const char *output_path,
|
|||
zupt_options_t *opts) {
|
||||
if (opts->block_size == 0) opts->block_size = auto_block_size(opts->level);
|
||||
|
||||
/* Resolve AUTO codec before compression */
|
||||
if (opts->codec_id == ZUPT_CODEC_AUTO)
|
||||
opts->codec_id = zupt_resolve_auto_codec();
|
||||
|
||||
FILE *out = fopen(output_path, "wb");
|
||||
if (!out) { fprintf(stderr, "Error: Cannot create '%s': %s\n", output_path, strerror(errno)); return ZUPT_ERR_IO; }
|
||||
|
||||
|
|
@ -729,6 +764,10 @@ zupt_error_t zupt_compress_solid(const char *output_path,
|
|||
if (opts->block_size == 0) opts->block_size = auto_block_size(opts->level);
|
||||
if (opts->block_size < 524288) opts->block_size = 524288;
|
||||
|
||||
/* Resolve AUTO codec before compression */
|
||||
if (opts->codec_id == ZUPT_CODEC_AUTO)
|
||||
opts->codec_id = zupt_resolve_auto_codec();
|
||||
|
||||
FILE *out = fopen(output_path, "wb");
|
||||
if (!out) { fprintf(stderr, "Error: Cannot create '%s'\n", output_path); return ZUPT_ERR_IO; }
|
||||
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ static void usage(void) {
|
|||
" -s, --store Store without compression\n"
|
||||
" -f, --fast Use fast LZ codec (less compression)\n"
|
||||
" --vv, --vaptvupt Use VaptVupt codec (fast LZ + ANS entropy)\n"
|
||||
" --lzhp Use Zupt-LZHP codec (LZ77+Huffman, no SIMD needed)\n"
|
||||
" -p, --password <PW> Encrypt with AES-256 (prompted if empty)\n"
|
||||
" -v, --verbose Verbose per-file output\n"
|
||||
" -t, --threads <N> Thread count (0=auto, 1=single, 2-64=explicit)\n"
|
||||
|
|
@ -144,6 +145,8 @@ int main(int argc, char **argv) {
|
|||
opts.codec_id=ZUPT_CODEC_ZUPT_LZ;
|
||||
} else if (streq(argv[ai],"--vv")||streq(argv[ai],"--vaptvupt")) {
|
||||
opts.codec_id=ZUPT_CODEC_VAPTVUPT; /* VAPTVUPT */
|
||||
} else if (streq(argv[ai],"--lzhp")) {
|
||||
opts.codec_id=ZUPT_CODEC_ZUPT_LZHP;
|
||||
} else if (streq(argv[ai],"-p")||streq(argv[ai],"--password")) {
|
||||
opts.encrypt=1;
|
||||
if (ai+1<argc && !isopt(argv[ai+1])) {
|
||||
|
|
@ -196,6 +199,10 @@ int main(int argc, char **argv) {
|
|||
opts.threads = 1;
|
||||
}
|
||||
|
||||
/* Resolve AUTO codec based on hardware detection */
|
||||
if (opts.codec_id == ZUPT_CODEC_AUTO)
|
||||
opts.codec_id = zupt_resolve_auto_codec();
|
||||
|
||||
fprintf(stderr, " Collected %d file(s) for compression%s\n", fl.count,
|
||||
opts.solid ? " (SOLID MODE)" : "");
|
||||
if (opts.threads > 1)
|
||||
|
|
|
|||
Loading…
Reference in a new issue