diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 5509a18..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,308 +0,0 @@ -name: CI - -on: - push: - branches: [main, master] - tags: ['v*'] - pull_request: - branches: [main, master] - -jobs: - # ═══════════════════════════════════════════════════════════════ - # Linux x86_64 — GCC + Clang, full test suite - # ═══════════════════════════════════════════════════════════════ - build-linux: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - compiler: [gcc, clang] - steps: - - uses: actions/checkout@v4 - - - name: Build - run: make CC=${{ matrix.compiler }} - - - name: Verify Jasmin symbols (x86_64) - run: | - nm zupt | grep -q "T zupt_mac_verify_ct" || exit 1 - nm zupt | grep -q "T zupt_ct_select_32" || exit 1 - nm zupt | grep -q "T zupt_fe_cswap" || exit 1 - nm zupt | grep -q "T zupt_aes256_blk" || exit 1 - nm zupt | grep -q "T zupt_aes256_ctr4" || exit 1 - echo "All 5 Jasmin symbols linked" - - - name: NIST/RFC test vectors (13 tests) - run: make test-vectors CC=${{ matrix.compiler }} && ./test_vectors - - - name: VaptVupt unit tests (11 tests) - run: make test-vv CC=${{ matrix.compiler }} - - - name: Regression tests (22 tests) - run: bash tests/regression.sh - - - name: Multi-threaded tests (14 tests) - run: bash tests/test_threaded.sh - - - name: Post-quantum tests (10 tests) - run: bash tests/test_pq.sh ./zupt - - - name: Benchmark - run: ./zupt bench --compare - - # ═══════════════════════════════════════════════════════════════ - # Linux x86_64 — ASAN + UBSan - # ═══════════════════════════════════════════════════════════════ - build-asan: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Build with ASAN + UBSan - run: make test-asan - - - name: ASAN smoke test (all codecs + encryption + PQ) - run: | - T=$(mktemp -d) - mkdir -p "$T/data/sub" - echo "ASAN test content" > "$T/data/hello.txt" - dd if=/dev/urandom bs=1024 count=200 of="$T/data/rand.bin" 2>/dev/null - seq 1 20000 > "$T/data/sub/numbers.txt" - yes "The quick brown fox. " | head -c 500000 > "$T/data/text.txt" - - # Normal compress + extract (VaptVupt auto) - ./zupt_asan compress "$T/a1.zupt" "$T/data/" 2>&1 - ./zupt_asan extract -o "$T/o1" "$T/a1.zupt" 2>&1 - - # Encrypted - ./zupt_asan compress -p "test123" "$T/a2.zupt" "$T/data/" 2>&1 - ./zupt_asan extract -o "$T/o2" -p "test123" "$T/a2.zupt" 2>&1 - - # Explicit LZHP codec - ./zupt_asan compress --lzhp "$T/a3.zupt" "$T/data/" 2>&1 - ./zupt_asan extract -o "$T/o3" "$T/a3.zupt" 2>&1 - - # Solid mode - ./zupt_asan compress --solid "$T/a4.zupt" "$T/data/" 2>&1 - ./zupt_asan extract -o "$T/o4" "$T/a4.zupt" 2>&1 - - # Multi-threaded - ./zupt_asan compress -t 4 "$T/a5.zupt" "$T/data/" 2>&1 - ./zupt_asan extract -o "$T/o5" "$T/a5.zupt" 2>&1 - - # PQ encryption - ./zupt_asan keygen -o "$T/priv.key" 2>&1 - ./zupt_asan keygen --pub -o "$T/pub.key" -k "$T/priv.key" 2>&1 - ./zupt_asan compress --pq "$T/pub.key" "$T/a6.zupt" "$T/data/" 2>&1 - ./zupt_asan extract --pq "$T/priv.key" -o "$T/o6" "$T/a6.zupt" 2>&1 - - # Integrity test - ./zupt_asan test "$T/a1.zupt" 2>&1 - - echo "ASAN: all modes clean" - rm -rf "$T" - - # ═══════════════════════════════════════════════════════════════ - # Linux aarch64 — cross-compile + QEMU - # ═══════════════════════════════════════════════════════════════ - build-linux-aarch64: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install aarch64 cross-compiler + QEMU - run: | - sudo apt-get update -q - sudo apt-get install -y -q gcc-aarch64-linux-gnu qemu-user-static - - - name: Build (aarch64, no Jasmin, no AVX2) - run: | - make CC=aarch64-linux-gnu-gcc \ - CFLAGS="-Wall -Wextra -O2 -std=c11 -Iinclude -Isrc -static" \ - LDFLAGS="-static" \ - LDLIBS="-lm -lpthread" - - - name: Verify no Jasmin symbols (aarch64) - run: | - ! nm zupt | grep -q "T zupt_aes256_blk" || \ - (echo "ERROR: Jasmin symbols should NOT be linked on aarch64" && exit 1) - echo "Correct: no Jasmin symbols on aarch64" - - - name: NIST/RFC test vectors (QEMU aarch64) - run: | - make test-vectors \ - CC=aarch64-linux-gnu-gcc \ - CFLAGS="-O2 -std=c11 -Iinclude -Isrc -static" \ - LDFLAGS="-static" \ - LDLIBS="-lm -lpthread" - qemu-aarch64-static ./test_vectors - - - name: VaptVupt unit tests (QEMU aarch64) - run: | - aarch64-linux-gnu-gcc -O2 -std=c11 -Iinclude -Isrc -static \ - tests/test_vaptvupt.c \ - src/vv_encoder.c src/vv_decoder.c src/vv_ans.c \ - src/vv_huffman.c src/vv_simd.c src/zupt_xxh.c src/zupt_cpuid.c \ - -lm -lpthread -o test_vaptvupt - qemu-aarch64-static ./test_vaptvupt - - - name: Smoke test (QEMU aarch64) - run: | - T=$(mktemp -d) - echo "aarch64 test" > "$T/test.txt" - seq 1 5000 >> "$T/test.txt" - qemu-aarch64-static ./zupt compress "$T/a.zupt" "$T/test.txt" 2>&1 - qemu-aarch64-static ./zupt extract -o "$T/out" "$T/a.zupt" 2>&1 - EXTR=$(find "$T/out" -name test.txt -type f | head -1) - diff -q "$T/test.txt" "$EXTR" - echo "aarch64 roundtrip: OK" - rm -rf "$T" - - # ═══════════════════════════════════════════════════════════════ - # macOS (Apple Silicon / Intel) - # ═══════════════════════════════════════════════════════════════ - build-macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - - name: Build - run: make - - - name: NIST/RFC test vectors - run: make test-vectors && ./test_vectors - - - name: VaptVupt unit tests - run: make test-vv - - - name: Regression tests - run: bash tests/regression.sh - - - name: Multi-threaded tests - run: bash tests/test_threaded.sh - - - name: Post-quantum tests - run: bash tests/test_pq.sh ./zupt - - # ═══════════════════════════════════════════════════════════════ - # Windows (MSYS2 / MinGW-w64) - # ═══════════════════════════════════════════════════════════════ - build-windows: - runs-on: windows-latest - defaults: - run: - shell: msys2 {0} - steps: - - uses: actions/checkout@v4 - - - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW64 - update: true - install: >- - mingw-w64-x86_64-gcc - make - diffutils - - - name: Build - run: make CC=gcc - - - name: NIST/RFC test vectors - run: make test-vectors && ./test_vectors - - - name: VaptVupt unit tests - run: make test-vv - - # ═══════════════════════════════════════════════════════════════ - # PIE / Hardening build (distro packaging compliance) - # ═══════════════════════════════════════════════════════════════ - build-hardened: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Build with PIE + full hardening - run: | - make CFLAGS="-Wall -Wextra -O2 -std=c11 -fPIE -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Iinclude -Isrc" \ - LDFLAGS="-pie -Wl,-z,relro,-z,now" - - - name: Verify PIE binary - run: | - file ./zupt | grep -q "pie executable" || \ - (echo "ERROR: binary is not PIE" && exit 1) - echo "PIE binary confirmed" - - - name: Test install with DESTDIR - run: | - make install DESTDIR=/tmp/zupt_pkg - test -f /tmp/zupt_pkg/usr/local/bin/zupt - test -f /tmp/zupt_pkg/usr/local/share/man/man1/zupt.1.gz - echo "Install layout OK" - - - name: Full test suite on hardened build - run: | - make test-vectors && ./test_vectors - make test-vv - bash tests/regression.sh - - # ═══════════════════════════════════════════════════════════════ - # Release — create GitHub release with tarball on tag push - # ═══════════════════════════════════════════════════════════════ - release: - if: startsWith(github.ref, 'refs/tags/v') - needs: [build-linux, build-asan, build-linux-aarch64, build-macos, build-windows, build-hardened] - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - - name: Build release tarball - run: | - VERSION="${GITHUB_REF_NAME#v}" - make clean - cd .. - cp -a zupt "zupt-${VERSION}" - tar czf "zupt-${VERSION}.tar.gz" "zupt-${VERSION}" \ - --exclude='*.o' --exclude='zupt-*/zupt' \ - --exclude='zupt_asan' --exclude='test_vectors' \ - --exclude='test_vaptvupt' --exclude='fuzz_*' - mv "zupt-${VERSION}.tar.gz" zupt/ - cd zupt - echo "TARBALL=zupt-${VERSION}.tar.gz" >> $GITHUB_ENV - echo "VERSION=${VERSION}" >> $GITHUB_ENV - - - name: Build binary - run: make - - - name: Run full test suite - run: | - make test-vectors && ./test_vectors - make test-vv - bash tests/regression.sh - bash tests/test_threaded.sh - bash tests/test_pq.sh ./zupt - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - name: "Zupt v${{ env.VERSION }}" - body: | - ## Zupt v${{ env.VERSION }} - - Backup compression with hardware-adaptive codec selection, AES-256 authenticated encryption, and post-quantum key encapsulation. - - **Changes:** See [CHANGELOG.md](https://github.com/cristiancmoises/zupt/blob/master/CHANGELOG.md) - - **Install:** - ```bash - curl -fsSL https://short.securityops.co/zupt | bash - ``` - Or build from source: - ```bash - tar xzf zupt-${{ env.VERSION }}.tar.gz && cd zupt-${{ env.VERSION }} && make && sudo make install - ``` - - **Test results:** 70/70 (11 VV + 13 NIST + 22 regression + 14 MT + 10 PQ) - files: ${{ env.TARBALL }} - generate_release_notes: true diff --git a/Makefile b/Makefile index a46ee71..2cb04b7 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,49 @@ # Zupt v2.0.0 — Makefile with VaptVupt codec + Jasmin integration -CC ?= gcc -CFLAGS ?= -Wall -Wextra -O2 -std=c11 -CFLAGS += -Iinclude -Isrc -LDLIBS = -lm -lpthread -PREFIX ?= /usr/local -BINDIR ?= $(PREFIX)/bin +# +# Multi-architecture: builds on x86_64, aarch64, armhf, ppc64le, s390x, riscv64. +# Jasmin CT crypto: x86_64 only (C fallback on all other architectures). +# AVX2 SIMD decode: x86_64 only (NEON on aarch64, scalar elsewhere). +# +# Packaging: +# make Build with defaults +# make V=1 Verbose build output +# make install DESTDIR=/ Staged install for packaging +# make CFLAGS="..." LDFLAGS="..." Override for distro policy (PIE, hardening) +# +# rpmlint / OBS compliance: +# - LDFLAGS honored on link line (before objects for PIE) +# - LDLIBS after objects (correct link order) +# - DESTDIR support for staged installs +# - Man page compressed and installed to $(MANDIR)/man1 -# ─── Zupt core sources ─── +CC ?= gcc +CFLAGS ?= -Wall -Wextra -O2 -std=c11 +CFLAGS += -Iinclude -Isrc +LDFLAGS ?= +LDLIBS ?= -lm -lpthread +PREFIX ?= /usr/local +BINDIR ?= $(PREFIX)/bin +MANDIR ?= $(PREFIX)/share/man +MAN1DIR ?= $(MANDIR)/man1 +GZIP ?= gzip +GZIPFLAGS ?= -9 -n + +# --- Verbose build --- +V ?= 0 +ifeq ($(V),1) + Q = +else + Q = @ +endif + +# --- Zupt core sources --- ZUPT_SOURCES = src/zupt_main.c src/zupt_format.c src/zupt_lz.c src/zupt_lzh.c \ src/zupt_xxh.c src/zupt_sha256.c src/zupt_aes256.c src/zupt_crypto.c \ src/zupt_predict.c src/zupt_parallel.c src/zupt_keccak.c \ src/zupt_x25519.c src/zupt_mlkem.c src/zupt_cpuid.c src/zupt_mlock.c \ src/zupt_filetype.c -# ─── VAPTVUPT: VaptVupt codec sources (Apache-2.0, integrated under MIT) ─── +# --- VAPTVUPT: VaptVupt codec sources (Apache-2.0, integrated under MIT) --- VV_SOURCES = src/vv_encoder.c src/vv_decoder.c src/vv_ans.c \ src/vv_huffman.c src/vv_simd.c @@ -21,77 +51,127 @@ SOURCES = $(ZUPT_SOURCES) $(VV_SOURCES) HEADERS = include/zupt.h include/zupt_keccak.h include/zupt_mlkem.h \ include/zupt_x25519.h include/zupt_cpuid.h include/zupt_jasmin.h \ + include/zupt_acsl.h \ include/vaptvupt.h include/vv_huffman.h include/vv_ans.h \ src/zupt_thread.h src/zupt_parallel.h -TARGET = zupt +TARGET = zupt +MANPAGE = doc/zupt.1 +MANPAGE_GZ = $(TARGET).1.gz + +# ═══════════════════════════════════════════════════════════════════ +# ARCHITECTURE DETECTION +# +# Jasmin CT assembly: x86_64 only (pre-compiled .s files) +# AVX2 SIMD decode: x86_64 only (-mavx2 on VV decode/encode/simd) +# NEON SIMD decode: aarch64 (auto-detected by compiler, no extra flags) +# Scalar fallback: all architectures +# ═══════════════════════════════════════════════════════════════════ -# ─── AVX2 detection: enable SIMD for VaptVupt on x86-64 ─── ARCH := $(shell uname -m) + +# --- AVX2: enable SIMD for VaptVupt on x86_64 --- ifeq ($(ARCH),x86_64) VV_SIMD_FLAGS = -mavx2 else VV_SIMD_FLAGS = endif -# ─── Jasmin: use pre-compiled .s files if present ─── -JAZZ_S = jasmin/zupt_mac_verify.s jasmin/zupt_mlkem_select.s jasmin/zupt_aes_ctr.s jasmin/zupt_x25519_fe.s jasmin/zupt_aes_ctr4.s -JAZZ_AVAILABLE := $(wildcard $(JAZZ_S)) +# --- Jasmin: enable only on x86_64 with pre-compiled .s files --- +JAZZ_S = jasmin/zupt_mac_verify.s jasmin/zupt_mlkem_select.s \ + jasmin/zupt_aes_ctr.s jasmin/zupt_x25519_fe.s jasmin/zupt_aes_ctr4.s +JAZZ_O = -ifeq ($(JAZZ_AVAILABLE),$(JAZZ_S)) - CFLAGS += -DZUPT_USE_JASMIN - JAZZ_O = jasmin/zupt_mac_verify.o jasmin/zupt_mlkem_select.o jasmin/zupt_aes_ctr.o jasmin/zupt_x25519_fe.o jasmin/zupt_aes_ctr4.o - $(info [jasmin] Verified assembly found — linking CT crypto) +ifeq ($(ARCH),x86_64) + JAZZ_AVAILABLE := $(wildcard $(JAZZ_S)) + ifeq ($(JAZZ_AVAILABLE),$(JAZZ_S)) + CFLAGS += -DZUPT_USE_JASMIN + JAZZ_O = jasmin/zupt_mac_verify.o jasmin/zupt_mlkem_select.o \ + jasmin/zupt_aes_ctr.o jasmin/zupt_x25519_fe.o jasmin/zupt_aes_ctr4.o + $(info [jasmin] Enabled (x86_64) — linking CT crypto) + else + $(info [jasmin] Assembly not found — using C fallback) + endif else - JAZZ_O = - $(info [jasmin] Assembly not found — using C fallback) + $(info [jasmin] Disabled on $(ARCH) — using C fallback) endif -# ─── Object files for per-file CFLAGS (VV SIMD files need -mavx2) ─── -VV_SIMD_OBJS = src/vv_encoder.o src/vv_decoder.o src/vv_simd.o +# --- Object files --- +# VV SIMD files need -mavx2 on x86_64 (no-op on other arches) +VV_SIMD_OBJS = src/vv_encoder.o src/vv_decoder.o src/vv_simd.o VV_PLAIN_OBJS = src/vv_ans.o src/vv_huffman.o -ZUPT_OBJS = $(patsubst %.c,%.o,$(ZUPT_SOURCES)) -ALL_OBJS = $(ZUPT_OBJS) $(VV_SIMD_OBJS) $(VV_PLAIN_OBJS) +ZUPT_OBJS = $(patsubst %.c,%.o,$(ZUPT_SOURCES)) +ALL_OBJS = $(ZUPT_OBJS) $(VV_SIMD_OBJS) $(VV_PLAIN_OBJS) + +# ═══════════════════════════════════════════════════════════════════ +# BUILD RULES +# ═══════════════════════════════════════════════════════════════════ .PHONY: all clean install uninstall test test-all test-asan test-vectors test-vv fuzz-build help all: $(TARGET) +# Jasmin pre-compiled assembly (x86_64 only) jasmin/%.o: jasmin/%.s - $(CC) -c -o $@ $< + $(Q)$(CC) $(CFLAGS) -c -o $@ $< -# VaptVupt SIMD files: compile with AVX2 +# VaptVupt SIMD files: compile with AVX2 on x86_64 $(VV_SIMD_OBJS): src/%.o: src/%.c $(HEADERS) - $(CC) $(CFLAGS) $(VV_SIMD_FLAGS) -c -o $@ $< + $(Q)$(CC) $(CFLAGS) $(VV_SIMD_FLAGS) -c -o $@ $< # VaptVupt non-SIMD files $(VV_PLAIN_OBJS): src/%.o: src/%.c $(HEADERS) - $(CC) $(CFLAGS) -c -o $@ $< + $(Q)$(CC) $(CFLAGS) -c -o $@ $< # Zupt core files $(ZUPT_OBJS): src/%.o: src/%.c $(HEADERS) - $(CC) $(CFLAGS) -c -o $@ $< + $(Q)$(CC) $(CFLAGS) -c -o $@ $< +# Link: LDFLAGS before objects (for PIE), LDLIBS after (for -lm -lpthread) $(TARGET): $(ALL_OBJS) $(JAZZ_O) - $(CC) $(CFLAGS) $(ALL_OBJS) $(JAZZ_O) $(LDLIBS) -o $(TARGET) - @echo "Build complete: ./$(TARGET)" + $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(ALL_OBJS) $(JAZZ_O) -o $(TARGET) $(LDLIBS) + @echo "Build complete: ./$(TARGET) [$(ARCH)]" -clean: - rm -f $(TARGET) zupt_asan test_vectors test_vaptvupt fuzz_decompress fuzz_vv_decompress jasmin/*.o src/*.o +# ═══════════════════════════════════════════════════════════════════ +# INSTALL / UNINSTALL +# ═══════════════════════════════════════════════════════════════════ install: $(TARGET) - @mkdir -p $(DESTDIR)$(BINDIR) - install -m 755 $(TARGET) $(DESTDIR)$(BINDIR)/$(TARGET) + $(Q)mkdir -p $(DESTDIR)$(BINDIR) + $(Q)install -m 755 $(TARGET) $(DESTDIR)$(BINDIR)/$(TARGET) + + $(Q)if [ -f "$(MANPAGE)" ]; then \ + mkdir -p $(DESTDIR)$(MAN1DIR); \ + $(GZIP) $(GZIPFLAGS) -c "$(MANPAGE)" > "$(DESTDIR)$(MAN1DIR)/$(MANPAGE_GZ)"; \ + chmod 0644 "$(DESTDIR)$(MAN1DIR)/$(MANPAGE_GZ)"; \ + echo "Installed: $(DESTDIR)$(MAN1DIR)/$(MANPAGE_GZ)"; \ + else \ + echo "Warning: man page not found: $(MANPAGE)"; \ + fi + @echo "Installed: $(DESTDIR)$(BINDIR)/$(TARGET)" uninstall: - rm -f $(DESTDIR)$(BINDIR)/$(TARGET) + $(Q)rm -f $(DESTDIR)$(BINDIR)/$(TARGET) + $(Q)rm -f $(DESTDIR)$(MAN1DIR)/$(MANPAGE_GZ) + +# ═══════════════════════════════════════════════════════════════════ +# CLEAN +# ═══════════════════════════════════════════════════════════════════ + +clean: + $(Q)rm -f $(TARGET) $(MANPAGE_GZ) zupt_asan test_vectors test_vaptvupt \ + fuzz_decompress fuzz_vv_decompress jasmin/*.o src/*.o + +# ═══════════════════════════════════════════════════════════════════ +# TEST TARGETS +# ═══════════════════════════════════════════════════════════════════ test: $(TARGET) - @sh tests/run_quick.sh + $(Q)sh tests/run_quick.sh test-all: $(TARGET) test-vectors test-vv - @echo "═══════════════════════════════════════════════" + @echo "===============================================" @sh tests/regression.sh 2>&1 | tail -3 @echo "" @sh tests/test_threaded.sh 2>&1 | tail -3 @@ -101,46 +181,61 @@ test-all: $(TARGET) test-vectors test-vv @./test_vectors 2>&1 | tail -2 @echo "" @./test_vaptvupt 2>&1 | tail -2 - @echo "═══════════════════════════════════════════════" + @echo "===============================================" test-vectors: tests/test_vectors.c $(HEADERS) - $(CC) -O2 -std=c11 -Iinclude -Isrc tests/test_vectors.c \ + $(Q)$(CC) -O2 -std=c11 -Iinclude -Isrc $(LDFLAGS) tests/test_vectors.c \ src/zupt_sha256.c src/zupt_crypto.c src/zupt_aes256.c src/zupt_xxh.c \ src/zupt_keccak.c src/zupt_x25519.c src/zupt_mlkem.c src/zupt_cpuid.c \ src/zupt_mlock.c \ - $(LDLIBS) -o test_vectors + -o test_vectors $(LDLIBS) # VAPTVUPT: VaptVupt codec unit tests test-vv: tests/test_vaptvupt.c $(HEADERS) - $(CC) $(CFLAGS) $(VV_SIMD_FLAGS) tests/test_vaptvupt.c \ + $(Q)$(CC) $(CFLAGS) $(VV_SIMD_FLAGS) $(LDFLAGS) tests/test_vaptvupt.c \ src/vv_encoder.c src/vv_decoder.c src/vv_ans.c src/vv_huffman.c \ src/vv_simd.c src/zupt_xxh.c src/zupt_cpuid.c \ - $(LDLIBS) -o test_vaptvupt - @./test_vaptvupt + -o test_vaptvupt $(LDLIBS) + $(Q)./test_vaptvupt test-asan: $(SOURCES) $(HEADERS) $(JAZZ_O) - $(CC) -Wall -Wextra -std=c11 -Iinclude -Isrc \ + $(Q)$(CC) -Wall -Wextra -std=c11 -Iinclude -Isrc \ -fsanitize=address,undefined -g -O1 \ - $(VV_SIMD_FLAGS) \ - $(SOURCES) $(JAZZ_O) $(LDLIBS) -o zupt_asan + $(VV_SIMD_FLAGS) $(LDFLAGS) \ + $(SOURCES) $(JAZZ_O) -o zupt_asan $(LDLIBS) @echo "ASAN build: ./zupt_asan" # AFL++ fuzzing harnesses (requires afl-clang-fast) fuzz-build: @echo "Building AFL++ fuzzing harnesses..." - afl-clang-fast -fsanitize=address,undefined -g -O1 -std=c11 \ - -Iinclude -Isrc $(VV_SIMD_FLAGS) \ + $(Q)afl-clang-fast -fsanitize=address,undefined -g -O1 -std=c11 \ + -Iinclude -Isrc $(VV_SIMD_FLAGS) $(LDFLAGS) \ $(filter-out src/zupt_main.c,$(SOURCES)) tests/fuzz_decompress.c \ - $(LDLIBS) -o fuzz_decompress - afl-clang-fast -fsanitize=address,undefined -g -O1 -std=c11 \ - -Iinclude -Isrc $(VV_SIMD_FLAGS) \ + -o fuzz_decompress $(LDLIBS) + $(Q)afl-clang-fast -fsanitize=address,undefined -g -O1 -std=c11 \ + -Iinclude -Isrc $(VV_SIMD_FLAGS) $(LDFLAGS) \ tests/fuzz_vv_decompress.c \ src/vv_encoder.c src/vv_decoder.c src/vv_ans.c src/vv_huffman.c \ src/vv_simd.c src/zupt_xxh.c src/zupt_cpuid.c \ - $(LDLIBS) -o fuzz_vv_decompress + -o fuzz_vv_decompress $(LDLIBS) @echo "Fuzz harnesses built. Run:" @echo " afl-fuzz -i corpus -o findings -- ./fuzz_decompress" @echo " afl-fuzz -i corpus_vv -o findings_vv -- ./fuzz_vv_decompress" help: - @echo "make / make test / make install / make test-all / make test-asan / make test-vv / make fuzz-build / make clean" + @echo "Zupt v2.0.0 build targets:" + @echo " make Build zupt binary" + @echo " make V=1 Build with verbose output" + @echo " make test Quick test" + @echo " make test-all Full test suite (regression + threaded + PQ + vectors + VV)" + @echo " make test-vv VaptVupt codec unit tests" + @echo " make test-asan Build with AddressSanitizer" + @echo " make fuzz-build Build AFL++ fuzzing harnesses" + @echo " make install Install to $(PREFIX)" + @echo " make uninstall Remove from $(PREFIX)" + @echo " make clean Remove build artifacts" + @echo "" + @echo "Architecture: $(ARCH)" + @echo " x86_64: Jasmin CT crypto + AVX2 SIMD decode" + @echo " aarch64: C crypto fallback + NEON SIMD decode" + @echo " other: C crypto fallback + scalar decode" diff --git a/include/vaptvupt.h b/include/vaptvupt.h index 6dc14d7..f7166c1 100644 --- a/include/vaptvupt.h +++ b/include/vaptvupt.h @@ -7,8 +7,6 @@ * VaptVupt Codec — Next-generation lossless compression * Public API and data structures * - * SPDX-License-Identifier: GPL-3.0-or-later - * Copyright 2026 Cristian. * Zero dependencies. Pure C11. */ #ifndef VAPTVUPT_H diff --git a/include/vv_ans.h b/include/vv_ans.h index 6cff4e7..352e6c5 100644 --- a/include/vv_ans.h +++ b/include/vv_ans.h @@ -101,7 +101,7 @@ vva_error_t vva_decode_ctx(const uint8_t *src, size_t src_len, */ #define VVA_ML_CODES 36 /* Match length code count */ -#define VVA_OF_CODES 24 /* Offset code count */ +#define VVA_OF_CODES 27 /* Offset code count: 3 rep + 24 explicit */ vva_error_t vva_encode_sequences(const uint8_t *tokens, size_t tok_len, uint8_t *dst, size_t dst_cap, size_t *dst_len, diff --git a/include/zupt.h b/include/zupt.h index 580964e..01c84ec 100644 --- a/include/zupt.h +++ b/include/zupt.h @@ -75,6 +75,7 @@ #define ZUPT_CODEC_ZUPT_LZH 0x0009 /* LZ77 + Huffman */ #define ZUPT_CODEC_ZUPT_LZHP 0x000A /* LZ77 + Huffman + Byte Prediction (default) */ #define ZUPT_CODEC_VAPTVUPT 0x0010 /* VAPTVUPT: VaptVupt LZ + ANS entropy codec */ +#define ZUPT_CODEC_AUTO 0xFFFF /* Auto-detect: VaptVupt if AVX2, else LZHP */ /* Crypto */ #define ZUPT_SALT_SIZE 32 @@ -326,4 +327,10 @@ const char *zupt_codec_name(uint16_t id); void zupt_default_options(zupt_options_t *o); void zupt_format_size(uint64_t bytes, char *buf, size_t cap); +/* Resolve ZUPT_CODEC_AUTO to a concrete codec based on hardware. + * On x86_64 with AVX2: VaptVupt (fast ANS+SIMD decode). + * On all other arches: Zupt-LZHP (no SIMD dependency). + * Decompression of ALL codecs works on ALL architectures. */ +uint16_t zupt_resolve_auto_codec(void); + #endif diff --git a/include/zupt_cpuid.h b/include/zupt_cpuid.h index 1cee10a..a508b5b 100644 --- a/include/zupt_cpuid.h +++ b/include/zupt_cpuid.h @@ -9,13 +9,15 @@ typedef struct { int has_aesni; /* CPUID.01H:ECX[25] — AES-NI instructions */ + int has_avx; /* AVX (VEX-encoded SSE) — requires CPUID + OS XSAVE */ int has_pclmul; /* CPUID.01H:ECX[1] — CLMUL (carry-less multiply) */ int has_avx2; /* CPUID.07H:EBX[5] — AVX2 (256-bit SIMD) */ int has_sse41; /* CPUID.01H:ECX[19] — SSE4.1 */ } zupt_cpu_features_t; -/*@ assigns f->has_aesni, f->has_pclmul, f->has_avx2, f->has_sse41; +/*@ assigns f->has_aesni, f->has_avx, f->has_pclmul, f->has_avx2, f->has_sse41; @ ensures f->has_aesni == 0 || f->has_aesni == 1; + @ ensures f->has_avx == 0 || f->has_avx == 1; @ ensures f->has_pclmul == 0 || f->has_pclmul == 1; @ ensures f->has_avx2 == 0 || f->has_avx2 == 1; @ ensures f->has_sse41 == 0 || f->has_sse41 == 1; diff --git a/src/vv_ans.c b/src/vv_ans.c index 4a4ee68..bbc8cec 100644 --- a/src/vv_ans.c +++ b/src/vv_ans.c @@ -6,6 +6,7 @@ #if !defined(_DEFAULT_SOURCE) && !defined(_GNU_SOURCE) #define _DEFAULT_SOURCE 1 #endif + /* * VaptVupt — tANS v2 (sparse header + 4-way interleaved decode) * @@ -1170,7 +1171,11 @@ static const uint8_t ml_extra[VVA_ML_CODES] = { 12,13,14,15 }; +/* Rep-match codes: 0=rep[0], 1=rep[1], 2=rep[2], 3+=explicit offset. + * Explicit offset code c (c≥3): offset in [2^(c-3), 2^(c-2)), (c-3) extra bits. + * This is how zstd encodes repeated offsets — saves 10-15 bits per rep-match. */ static const uint8_t of_extra[VVA_OF_CODES] = { + 0,0,0, /* rep codes: 0 extra bits */ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23 }; @@ -1192,21 +1197,24 @@ static uint32_t ml_decode(uint8_t code, uint32_t extra) { return ml_base[code] + extra; } -/* Encode offset → (code, extra_value, extra_bits) */ +/* Encode explicit offset → (code, extra_value, extra_bits). + * Returns code in range [3..26]. Caller handles rep-match codes 0-2. */ static void of_encode(uint32_t offset, uint8_t *code, uint32_t *extra, int *nbits) { - if (offset == 0) { *code = 0; *extra = 0; *nbits = 0; return; } + if (offset == 0) { *code = 3; *extra = 0; *nbits = 0; return; } int c = 0; uint32_t v = offset; while (v > 1) { v >>= 1; c++; } - if (c >= VVA_OF_CODES) c = VVA_OF_CODES - 1; - *code = (uint8_t)c; + if (c >= 24) c = 23; /* clamp to 24 explicit codes */ + *code = (uint8_t)(c + 3); /* shift by 3 for rep codes */ *extra = offset - (1u << c); - *nbits = of_extra[c]; + *nbits = of_extra[c + 3]; } -/* Decode offset code → offset */ +/* Decode offset code → offset. Codes 0-2 are rep-match (caller resolves). + * Codes 3-26 are explicit offsets. */ static uint32_t of_decode(uint8_t code, uint32_t extra) { - return (1u << code) + extra; + if (code < 3) return 0; /* rep-match — caller must handle */ + return (1u << (code - 3)) + extra; } /* Write a varint to a buffer, return bytes written */ @@ -1349,21 +1357,61 @@ vva_error_t vva_encode_sequences(const uint8_t *tokens, size_t tok_len, } } - /* ─── Count ML and OF code frequencies ─── */ + /* ─── Count ML and OF code frequencies with rep-match tracking ─── */ uint32_t freq_ml[VVA_ML_CODES], freq_of[VVA_OF_CODES]; memset(freq_ml, 0, sizeof(freq_ml)); memset(freq_of, 0, sizeof(freq_of)); + /* Precompute OF codes with rep-match detection (forward pass). + * Store in per-sequence arrays so the backward ANS pass can use them. */ + uint8_t *seq_of_code = NULL; + uint32_t *seq_of_extra = NULL; + int *seq_of_nbits = NULL; + seq_of_code = (uint8_t *)malloc(nseq * sizeof(uint8_t)); + seq_of_extra = (uint32_t *)malloc(nseq * sizeof(uint32_t)); + seq_of_nbits = (int *)malloc(nseq * sizeof(int)); + if (!seq_of_code || !seq_of_extra || !seq_of_nbits) { + free(seq_of_code); free(seq_of_extra); free(seq_of_nbits); + free(seqs); free(lit_buf); free(lit_enc); + return VVA_ERR_NOMEM; + } + size_t match_count = 0; + uint32_t enc_rep[3] = {0, 0, 0}; /* Rep-match tracking during forward pass */ for (size_t i = 0; i < nseq; i++) { if (seqs[i].matchlen > 0) { uint8_t mc; uint32_t mx; int mn; ml_encode(seqs[i].matchlen, &mc, &mx, &mn); freq_ml[mc]++; + + /* Check rep-match before explicit encoding */ + uint32_t off = seqs[i].offset; uint8_t oc; uint32_t ox; int on; - of_encode(seqs[i].offset, &oc, &ox, &on); + if (off == enc_rep[0] && off != 0) { + oc = 0; ox = 0; on = 0; /* rep[0] */ + } else if (off == enc_rep[1] && off != 0) { + oc = 1; ox = 0; on = 0; /* rep[1] */ + } else if (off == enc_rep[2] && off != 0) { + oc = 2; ox = 0; on = 0; /* rep[2] */ + } else { + of_encode(off, &oc, &ox, &on); /* explicit: codes 3-26 */ + } + seq_of_code[i] = oc; + seq_of_extra[i] = ox; + seq_of_nbits[i] = on; freq_of[oc]++; + + /* Update rep array (same logic as LZ engine) */ + if (off != enc_rep[0] && off != 0) { + enc_rep[2] = enc_rep[1]; + enc_rep[1] = enc_rep[0]; + enc_rep[0] = off; + } match_count++; + } else { + seq_of_code[i] = 0; + seq_of_extra[i] = 0; + seq_of_nbits[i] = 0; } } @@ -1440,11 +1488,15 @@ vva_error_t vva_encode_sequences(const uint8_t *tokens, size_t tok_len, for (size_t ii = nseq; ii > 0; ii--) { if (seqs[ii - 1].matchlen == 0) continue; - uint8_t mc, oc; - uint32_t mx, ox; - int mn, on; + uint8_t mc; + uint32_t mx; + int mn; ml_encode(seqs[ii - 1].matchlen, &mc, &mx, &mn); - of_encode(seqs[ii - 1].offset, &oc, &ox, &on); + + /* Use precomputed OF code from forward pass (rep-match aware) */ + uint8_t oc = seq_of_code[ii - 1]; + uint32_t ox = seq_of_extra[ii - 1]; + int on = seq_of_nbits[ii - 1]; /* Encode in this order (reversed): ml_code, ml_extra, of_code, of_extra * Decoder reads: of_extra, of_code, ml_extra, ml_code */ @@ -1513,7 +1565,15 @@ vva_error_t vva_encode_sequences(const uint8_t *tokens, size_t tok_len, } /* ─── Encode litlen varints ─── */ - litlen_buf = (uint8_t *)malloc(nseq * 5 + 1); + /* Each litlen varint uses ceil(litlen/255)+1 bytes. Worst case for + * nseq sequences each with large litlen: compute exact bound. */ + { + size_t litlen_cap = nseq; /* at least 1 byte per sequence */ + for (size_t i = 0; i < nseq; i++) + litlen_cap += seqs[i].litlen / 255; + litlen_cap += 16; /* safety margin */ + litlen_buf = (uint8_t *)malloc(litlen_cap); + } if (!litlen_buf) goto seq_fail; { size_t pos = 0; @@ -1572,11 +1632,13 @@ vva_error_t vva_encode_sequences(const uint8_t *tokens, size_t tok_len, } free(seqs); free(lit_buf); free(lit_enc); + free(seq_of_code); free(seq_of_extra); free(seq_of_nbits); free(ml_hdr_buf); free(of_hdr_buf); free(seq_bs); free(litlen_buf); return VVA_OK; seq_fail: free(seqs); free(lit_buf); free(lit_enc); + free(seq_of_code); free(seq_of_extra); free(seq_of_nbits); free(ml_hdr_buf); free(of_hdr_buf); free(seq_bs); free(litlen_buf); return VVA_ERR_OVERFLOW; } @@ -1691,15 +1753,11 @@ vva_error_t vva_decode_sequences(const uint8_t *src, size_t src_len, uint8_t *op_end = dst + dst_cap; size_t lit_pos = 0; size_t matches_decoded = 0; + uint32_t dec_rep[3] = {0, 0, 0}; /* Rep-match offset tracking */ - /* We don't know exact nseq, but we have match_count matches + possibly - * one final literal-only sequence. Decode until all literals consumed - * and all matches decoded. */ while (lit_pos < total_lits || matches_decoded < match_count) { - /* Read litlen from varint stream */ size_t litlen = seq_read_varint(&ll_p, end); - /* Copy literals from decoded literal buffer */ if (lit_pos + litlen > total_lits) { free(dec_ml); free(dec_of); free(lit_buf); return VVA_ERR_CORRUPT; } if (op + litlen > op_end) { free(dec_ml); free(dec_of); free(lit_buf); return VVA_ERR_OVERFLOW; } if (litlen > 0) { @@ -1708,23 +1766,31 @@ vva_error_t vva_decode_sequences(const uint8_t *src, size_t src_len, lit_pos += litlen; } - /* If all matches decoded, this was the last literal-only sequence */ if (matches_decoded >= match_count) break; - /* Decode offset */ + /* Decode OF code */ if (r.n < ANS_LOG) br_fill(&r); if (state_of >= (uint32_t)ANS_L) { free(dec_ml); free(dec_of); free(lit_buf); return VVA_ERR_CORRUPT; } vva_dec_entry_t eof = dec_of[state_of]; uint32_t of_bits = br_read(&r, eof.nbits); state_of = (uint32_t)eof.baseline + of_bits; - /* Read offset extra bits */ + /* Resolve offset: codes 0-2 = rep-match, 3+ = explicit */ uint8_t of_code = eof.symbol; - uint32_t of_extra_val = 0; - if (of_code < VVA_OF_CODES && of_extra[of_code] > 0) { - of_extra_val = br_read(&r, of_extra[of_code]); + uint32_t offset; + if (of_code < 3) { + offset = dec_rep[of_code]; + } else { + uint32_t of_extra_val = 0; + if (of_code < VVA_OF_CODES && of_extra[of_code] > 0) { + of_extra_val = br_read(&r, of_extra[of_code]); + } + offset = of_decode(of_code, of_extra_val); + } + /* Update rep offsets */ + if (offset != 0 && offset != dec_rep[0]) { + dec_rep[2] = dec_rep[1]; dec_rep[1] = dec_rep[0]; dec_rep[0] = offset; } - uint32_t offset = of_decode(of_code, of_extra_val); /* Decode match length */ if (r.n < ANS_LOG) br_fill(&r); diff --git a/src/vv_decoder.c b/src/vv_decoder.c index e58f340..a273dae 100644 --- a/src/vv_decoder.c +++ b/src/vv_decoder.c @@ -6,6 +6,7 @@ #if !defined(_DEFAULT_SOURCE) && !defined(_GNU_SOURCE) #define _DEFAULT_SOURCE 1 #endif + /* * VaptVupt — Decoder v2 (Sprint 1) * diff --git a/src/vv_encoder.c b/src/vv_encoder.c index 1890500..05336d5 100644 --- a/src/vv_encoder.c +++ b/src/vv_encoder.c @@ -6,6 +6,7 @@ #if !defined(_DEFAULT_SOURCE) && !defined(_GNU_SOURCE) #define _DEFAULT_SOURCE 1 #endif + /* * VaptVupt — Encoder v2 (Sprint 1) * @@ -442,11 +443,8 @@ int64_t vv_compress(const uint8_t *src, size_t src_len, if (wlog == 0) { switch (opts->mode) { case VV_MODE_ULTRA_FAST: wlog = 16; break; - case VV_MODE_BALANCED: wlog = 16; break; - case VV_MODE_EXTREME: wlog = 16; break; - /* TRADEOFF: wlog=16 default avoids 3-byte offset overhead on small data. - * Users can set opts.window_log=20 (1MB) or 22 (4MB) for large files - * with long-range patterns. Zupt sets wlog=20 for backup chunks >1MB. */ + case VV_MODE_BALANCED: wlog = 16; break; /* may be overridden below */ + case VV_MODE_EXTREME: wlog = 16; break; /* may be overridden below */ } } switch (opts->mode) { @@ -456,6 +454,41 @@ int64_t vv_compress(const uint8_t *src, size_t src_len, default: depth = 48; } + /* ─── ADAPTIVE WINDOW (Item 2): trial-compress first block at wlog=16 + * and wlog=20. If wlog=20 produces ≥3% smaller output, use it. + * Only for balanced/extreme with auto wlog (opts->window_log == 0). + * Cost: one extra compression of the first block (~10ms for 1MB). + * TRADEOFF: encode speed vs automatic ratio optimization. + * Zupt benefits because backup data characteristics are unknown. ─── */ + if (opts->window_log == 0 && opts->mode >= VV_MODE_BALANCED && src_len > 65536) { + size_t trial_len = src_len; + if (trial_len > VV_MAX_BLOCK_SIZE) trial_len = VV_MAX_BLOCK_SIZE; + + size_t trial_cap = trial_len + trial_len / 255 + 1024; + uint8_t *trial_buf = (uint8_t *)malloc(trial_cap); + if (trial_buf) { + /* Trial at wlog=16 */ + matcher_t m16; + matcher_init(&m16, 16, depth); + size_t sz16 = compress_block(src, trial_len, trial_buf, trial_cap, &m16, opts->mode); + matcher_free(&m16); + + /* Trial at wlog=20 */ + matcher_t m20; + matcher_init(&m20, 20, depth); + size_t sz20 = compress_block(src, trial_len, trial_buf, trial_cap, &m20, opts->mode); + matcher_free(&m20); + + free(trial_buf); + + /* Pick winner: wlog=20 must save ≥3% to justify 3-byte offsets */ + if (sz20 > 0 && sz16 > 0 && sz20 < (sz16 * 97 / 100)) { + wlog = 20; + } + /* Otherwise stay at wlog=16 (no regression on short-offset data) */ + } + } + /* Frame header */ uint8_t *op = dst; vv_frame_header_t fh; diff --git a/src/vv_huffman.c b/src/vv_huffman.c index 2deb892..b7885b9 100644 --- a/src/vv_huffman.c +++ b/src/vv_huffman.c @@ -6,6 +6,7 @@ #if !defined(_DEFAULT_SOURCE) && !defined(_GNU_SOURCE) #define _DEFAULT_SOURCE 1 #endif + /* * VaptVupt — Canonical Huffman Codec Implementation * diff --git a/src/vv_simd.c b/src/vv_simd.c index 0b31724..ba06aba 100644 --- a/src/vv_simd.c +++ b/src/vv_simd.c @@ -6,6 +6,7 @@ #if !defined(_DEFAULT_SOURCE) && !defined(_GNU_SOURCE) #define _DEFAULT_SOURCE 1 #endif + /* * VaptVupt — SIMD-accelerated copy routines * @@ -38,8 +39,8 @@ static void copy_match_scalar(uint8_t *dst, uint32_t offset, size_t length) { dst += 16; src += 16; length -= 16; } if (length > 0) memcpy(dst, src, length); - } else if (offset >= 4) { - /* Moderate overlap: 8-byte copy with re-read */ + } else if (offset >= 8) { + /* Moderate overlap (8-15): 8-byte copy is safe since offset >= stride */ while (length >= 8) { uint64_t v; memcpy(&v, src, 8); @@ -48,7 +49,7 @@ static void copy_match_scalar(uint8_t *dst, uint32_t offset, size_t length) { } while (length-- > 0) *dst++ = *src++; } else { - /* Very short overlap (1-3): byte-by-byte */ + /* Short overlap (1-7): byte-by-byte to handle pattern repeat correctly */ for (size_t i = 0; i < length; i++) dst[i] = src[i]; } } diff --git a/src/zupt_cpuid.c b/src/zupt_cpuid.c index 423c967..2845562 100644 --- a/src/zupt_cpuid.c +++ b/src/zupt_cpuid.c @@ -9,7 +9,7 @@ #include /* Global instance */ -zupt_cpu_features_t zupt_cpu = {0, 0, 0, 0}; +zupt_cpu_features_t zupt_cpu = {0, 0, 0, 0, 0}; /* ═══════════════════════════════════════════════════════════════════ * CPUID intrinsics — platform-specific @@ -30,6 +30,9 @@ zupt_cpu_features_t zupt_cpu = {0, 0, 0, 0}; __cpuidex(regs, leaf, subleaf); *eax = regs[0]; *ebx = regs[1]; *ecx = regs[2]; *edx = regs[3]; } + static uint64_t zupt_xgetbv(uint32_t idx) { + return _xgetbv(idx); + } #elif defined(__GNUC__) || defined(__clang__) #include static void zupt_cpuid(int leaf, int subleaf, int *eax, int *ebx, int *ecx, int *edx) { @@ -37,6 +40,11 @@ zupt_cpu_features_t zupt_cpu = {0, 0, 0, 0}; __cpuid_count((unsigned int)leaf, (unsigned int)subleaf, a, b, c, d); *eax = (int)a; *ebx = (int)b; *ecx = (int)c; *edx = (int)d; } + static uint64_t zupt_xgetbv(uint32_t idx) { + uint32_t lo, hi; + __asm__ __volatile__ ("xgetbv" : "=a"(lo), "=d"(hi) : "c"(idx)); + return ((uint64_t)hi << 32) | lo; + } #else /* Inline assembly fallback */ static void zupt_cpuid(int leaf, int subleaf, int *eax, int *ebx, int *ecx, int *edx) { @@ -46,6 +54,11 @@ zupt_cpu_features_t zupt_cpu = {0, 0, 0, 0}; : "a"(leaf), "c"(subleaf) ); } + static uint64_t zupt_xgetbv(uint32_t idx) { + uint32_t lo, hi; + __asm__ __volatile__ ("xgetbv" : "=a"(lo), "=d"(hi) : "c"(idx)); + return ((uint64_t)hi << 32) | lo; + } #endif void zupt_detect_cpu(zupt_cpu_features_t *f) { @@ -62,11 +75,27 @@ void zupt_detect_cpu(zupt_cpu_features_t *f) { f->has_aesni = (ecx >> 25) & 1; /* ECX bit 25 */ f->has_pclmul = (ecx >> 1) & 1; /* ECX bit 1 */ f->has_sse41 = (ecx >> 19) & 1; /* ECX bit 19 */ + + /* AVX detection: CPU must support AVX (ECX[28]) AND the OS must + * have enabled XSAVE/XRSTOR for YMM state (OSXSAVE, ECX[27]). + * If OSXSAVE is set, check XCR0 bits 1+2 (SSE+AVX state). + * Without this check, VEX-encoded instructions (vaesenc, vmovdqu, + * vpxor, etc.) will SIGILL even if the CPU supports them. */ + int has_avx_cpu = (ecx >> 28) & 1; + int has_osxsave = (ecx >> 27) & 1; + if (has_avx_cpu && has_osxsave) { + uint64_t xcr0 = zupt_xgetbv(0); + /* Bits 1 (SSE/XMM) and 2 (AVX/YMM) must both be set */ + if ((xcr0 & 0x6) == 0x6) + f->has_avx = 1; + } } if (max_leaf >= 7) { zupt_cpuid(7, 0, &eax, &ebx, &ecx, &edx); - f->has_avx2 = (ebx >> 5) & 1; /* EBX bit 5 */ + /* AVX2 also requires AVX (OS XSAVE) to be usable */ + if (f->has_avx && ((ebx >> 5) & 1)) + f->has_avx2 = 1; } } @@ -74,7 +103,7 @@ void zupt_detect_cpu(zupt_cpu_features_t *f) { void zupt_detect_cpu(zupt_cpu_features_t *f) { memset(f, 0, sizeof(*f)); - /* No AES-NI on ARM/RISC-V/etc — use table fallback */ + /* No AES-NI/AVX on ARM/RISC-V/etc — use table fallback */ } #endif /* ZUPT_HAS_CPUID */ diff --git a/src/zupt_crypto.c b/src/zupt_crypto.c index 5da202f..c9ce2d4 100644 --- a/src/zupt_crypto.c +++ b/src/zupt_crypto.c @@ -189,9 +189,11 @@ void zupt_aes256_ctr(const uint8_t key[32], const uint8_t nonce[16], #ifdef ZUPT_USE_JASMIN /* JASMIN-VERIFIED: AES-NI path — constant-time, no T-table leakage. - * Requires AES-NI support (detected via CPUID at startup). - * Uses 4-block pipeline for bulk data, single-block for tail. */ - if (zupt_cpu.has_aesni) { + * The Jasmin-generated assembly uses VEX-encoded instructions (vaesenc, + * vmovdqu, vpxor, etc.) which require BOTH AES-NI AND AVX support. + * Checking only has_aesni would SIGILL on CPUs with AES-NI but no AVX, + * or where the OS hasn't enabled XSAVE for YMM state. */ + if (zupt_cpu.has_aesni && zupt_cpu.has_avx) { size_t full_blocks = len / 16; size_t tail_bytes = len % 16; diff --git a/src/zupt_format.c b/src/zupt_format.c index ae99609..7a5970f 100644 --- a/src/zupt_format.c +++ b/src/zupt_format.c @@ -9,6 +9,7 @@ */ #define _GNU_SOURCE #include "zupt.h" +#include "zupt_cpuid.h" /* zupt_cpu for AUTO codec detection */ #include "zupt_parallel.h" #include "vaptvupt.h" /* VAPTVUPT: VaptVupt codec integration */ #include @@ -51,6 +52,7 @@ const char *zupt_codec_name(uint16_t id) { case ZUPT_CODEC_ZUPT_LZH: return "Zupt-LZH"; case ZUPT_CODEC_ZUPT_LZHP: return "Zupt-LZHP"; case ZUPT_CODEC_VAPTVUPT: return "VaptVupt"; /* VAPTVUPT */ + case ZUPT_CODEC_AUTO: return "Auto"; default: return "Unknown"; } } @@ -58,12 +60,41 @@ void zupt_default_options(zupt_options_t *o) { memset(o, 0, sizeof(*o)); o->level = 7; o->block_size = 0; - o->codec_id = ZUPT_CODEC_VAPTVUPT; /* VAPTVUPT: default codec v2.0.0 */ + o->codec_id = ZUPT_CODEC_AUTO; /* Auto-detect: VaptVupt if AVX2, else LZHP */ /* Init keyring canaries */ o->keyring.canary_head = ZUPT_CANARY; o->keyring.canary_tail = ZUPT_CANARY; } +/* Resolve ZUPT_CODEC_AUTO to a concrete codec. + * VaptVupt decode works on ALL architectures (scalar fallback), but the + * AVX2 SIMD decode path gives ~3× throughput. On non-AVX2 hardware, + * Zupt-LZHP is a better default since its simpler decoder doesn't + * benefit from SIMD as much. + * + * Detection order: + * 1. Compile-time: __x86_64__ + __AVX2__ → VaptVupt (compiled with -mavx2) + * 2. Runtime: zupt_cpu.has_avx2 → VaptVupt (for x86_64 without -mavx2) + * 3. Compile-time: __aarch64__ + __ARM_NEON → VaptVupt (NEON decode) + * 4. Fallback: Zupt-LZHP (works everywhere) + */ +uint16_t zupt_resolve_auto_codec(void) { +#if defined(__x86_64__) || defined(_M_X64) + /* x86_64: check AVX2 at compile time (via -mavx2) or runtime (cpuid) */ + #if defined(__AVX2__) + return ZUPT_CODEC_VAPTVUPT; /* Compiled with -mavx2: inline SIMD decode */ + #else + if (zupt_cpu.has_avx2) + return ZUPT_CODEC_VAPTVUPT; /* Runtime AVX2: vv_simd.c dispatch */ + return ZUPT_CODEC_ZUPT_LZHP; /* No AVX2: use LZHP */ + #endif +#elif defined(__aarch64__) && defined(__ARM_NEON) + return ZUPT_CODEC_VAPTVUPT; /* NEON SIMD decode available */ +#else + return ZUPT_CODEC_ZUPT_LZHP; /* Scalar only: LZHP is a better default */ +#endif +} + static uint32_t auto_block_size(int level) { if (level <= 2) return 131072; if (level <= 4) return 131072; @@ -276,6 +307,10 @@ zupt_error_t zupt_compress_files(const char *output_path, zupt_options_t *opts) { if (opts->block_size == 0) opts->block_size = auto_block_size(opts->level); + /* Resolve AUTO codec before compression */ + if (opts->codec_id == ZUPT_CODEC_AUTO) + opts->codec_id = zupt_resolve_auto_codec(); + FILE *out = fopen(output_path, "wb"); if (!out) { fprintf(stderr, "Error: Cannot create '%s': %s\n", output_path, strerror(errno)); return ZUPT_ERR_IO; } @@ -729,6 +764,10 @@ zupt_error_t zupt_compress_solid(const char *output_path, if (opts->block_size == 0) opts->block_size = auto_block_size(opts->level); if (opts->block_size < 524288) opts->block_size = 524288; + /* Resolve AUTO codec before compression */ + if (opts->codec_id == ZUPT_CODEC_AUTO) + opts->codec_id = zupt_resolve_auto_codec(); + FILE *out = fopen(output_path, "wb"); if (!out) { fprintf(stderr, "Error: Cannot create '%s'\n", output_path); return ZUPT_ERR_IO; } diff --git a/src/zupt_main.c b/src/zupt_main.c index 21864af..1cba67a 100644 --- a/src/zupt_main.c +++ b/src/zupt_main.c @@ -48,6 +48,7 @@ static void usage(void) { " -s, --store Store without compression\n" " -f, --fast Use fast LZ codec (less compression)\n" " --vv, --vaptvupt Use VaptVupt codec (fast LZ + ANS entropy)\n" + " --lzhp Use Zupt-LZHP codec (LZ77+Huffman, no SIMD needed)\n" " -p, --password Encrypt with AES-256 (prompted if empty)\n" " -v, --verbose Verbose per-file output\n" " -t, --threads Thread count (0=auto, 1=single, 2-64=explicit)\n" @@ -144,6 +145,8 @@ int main(int argc, char **argv) { opts.codec_id=ZUPT_CODEC_ZUPT_LZ; } else if (streq(argv[ai],"--vv")||streq(argv[ai],"--vaptvupt")) { opts.codec_id=ZUPT_CODEC_VAPTVUPT; /* VAPTVUPT */ + } else if (streq(argv[ai],"--lzhp")) { + opts.codec_id=ZUPT_CODEC_ZUPT_LZHP; } else if (streq(argv[ai],"-p")||streq(argv[ai],"--password")) { opts.encrypt=1; if (ai+1 1)