From 162cf462b6e3ad0d851728442bbd7e70eac91c9c Mon Sep 17 00:00:00 2001 From: Eremey Valetov Date: Mon, 30 Mar 2026 16:57:47 -0400 Subject: [PATCH] Fix CI failures and formatting issues - Mark test corpus/archives as binary in .gitattributes to prevent line ending conversion on CI (fixes extract test size mismatch) - Fix alignment-unsafe struct cast in uc2_dict.c serialize/deserialize (use memcpy-based byte access instead; fixes SEGFAULT on CI) - Fix formatting issues in docs --- .gitattributes | 4 +++ CREDITS.md | 12 ++++---- README.md | 36 +++++++++++------------ ROADMAP.md | 14 ++++----- docs/roadmap.rst | 20 ++++++------- lib/src/uc2_dict.c | 71 +++++++++++++++++++++++++--------------------- 6 files changed, 84 insertions(+), 73 deletions(-) diff --git a/.gitattributes b/.gitattributes index 950cfc9..518f6aa 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,3 +4,7 @@ original/UC2_source/**/*.exe binary original/UC2_source/**/*.DAT binary original/UC2_source/**/*.LIB binary original/UC2_source/**/*.PRJ binary + +# Test corpus and archives must be byte-exact +tests/corpus/** binary +tests/archives/** binary diff --git a/CREDITS.md b/CREDITS.md index 33f4eb1..a8fe168 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -2,7 +2,7 @@ ## Original UltraCompressor II -**Nico de Vries** created UltraCompressor II (1992–1996), a DOS archiver +**Nico de Vries** created UltraCompressor II (1992--1996), a DOS archiver with LZ77+Huffman compression, master-block deduplication, and file versioning. The original source code is preserved in `original/UC2_source/`. @@ -15,7 +15,7 @@ versioning. The original source code is preserved in `original/UC2_source/`. ## Portable Decompressor (unuc2 / libunuc2) -**Jan Bobrowski** wrote a clean-room portable decompressor (2020–2021) that +**Jan Bobrowski** wrote a clean-room portable decompressor (2020--2021) that forms the foundation of this project's decompression engine. The library (`libunuc2`) is licensed under LGPL-3.0; the CLI tool (`unuc2`) is licensed under GPL-3.0. @@ -25,17 +25,17 @@ under GPL-3.0. ## Additional Contributors -- **Jan-Pieter Cornet** — early testing, archive samples, and format +- **Jan-Pieter Cornet** -- early testing, archive samples, and format documentation contributions to the unuc2 project. -- **Vladislav Sagunov** — maintained UC2 resources and documentation. +- **Vladislav Sagunov** -- maintained UC2 resources and documentation. ## UC2 v3.0.0 Revival -**Eremey Valetov** — project revival, including: +**Eremey Valetov** -- project revival, including: - CMake build system and cross-platform porting (Linux, macOS, MSVC, DJGPP) - LZ77+Huffman compression engine (compatible with original UC2 Pro) -- rANS entropy coder (method 10, levels 6–9) +- rANS entropy coder (method 10, levels 6--9) - Content-defined chunking (CDC) with Gear rolling hash - Merkle DAG content addressing - Cross-archive block store for shared deduplication diff --git a/README.md b/README.md index 9a2dc31..e5663b2 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,18 @@ -# UC2 v3.0.0 — UltraCompressor II +# UC2 v3.0.0 -- UltraCompressor II A cross-platform revival of UltraCompressor II, the DOS-era archiver by -Nico de Vries (1992–1996). UC2 was notable for its advanced deduplication +Nico de Vries (1992--1996). UC2 was notable for its advanced deduplication ("master blocks"), file versioning, and competitive compression ratios on the hardware of its day. UC2 v3 brings it back as a modern, portable C99 tool with full -backward compatibility — archives created by UC2 v3 can be extracted +backward compatibility -- archives created by UC2 v3 can be extracted by the original 1992 UC2 Pro, and vice versa. ## Features -- **Full compression and decompression** — LZ77+Huffman (levels 2–5), - rANS entropy coding (levels 6–9), LZ4 ultra-fast mode +- **Full compression and decompression** -- LZ77+Huffman (levels 2--5), + rANS entropy coding (levels 6--9), LZ4 ultra-fast mode - **Backward compatible** with the original UC2 Pro (verified via automated DOSBox-X cross-tool round-trip testing) - **Content-defined chunking** (CDC) with Gear rolling hash for @@ -21,16 +21,16 @@ by the original 1992 UC2 Pro, and vice versa. - **Cross-archive dedup** via shared block store - **Near-duplicate detection** via SimHash - **Delta compression** for binary patching -- **Content-aware preprocessing** — BCJ (x86), BWT (text), delta filter +- **Content-aware preprocessing** -- BCJ (x86), BWT (text), delta filter - **BLAKE3** cryptographic hashing for archive integrity -- **Benchmark mode** — test all methods on input data -- **Personality** — warm, confident status messages (`-q` for scripting) +- **Benchmark mode** -- test all methods on input data +- **Personality** -- warm, confident status messages (`-q` for scripting) - Directory archival with nested hierarchies - Cross-platform: Linux, macOS, Windows (MSVC), DOS (DJGPP) ## Building -Requires CMake ≥ 3.16 and a C99 compiler (GCC, Clang, or MSVC). +Requires CMake >= 3.16 and a C99 compiler (GCC, Clang, or MSVC). ```sh cmake -B build @@ -63,8 +63,8 @@ uc2 -B files... # Benchmark all methods | 8 | rANS | Tight | | 9 | rANS | Ultra | -Levels 2–5 produce archives readable by the original 1992 UC2 Pro. -Levels 6–9 use rANS entropy coding (UC2 v3 only, better compression). +Levels 2--5 produce archives readable by the original 1992 UC2 Pro. +Levels 6--9 use rANS entropy coding (UC2 v3 only, better compression). ### Options @@ -73,7 +73,7 @@ Levels 6–9 use rANS entropy coding (UC2 v3 only, better compression). | `-w` | Create archive | | `-l` | List archive contents | | `-t` | Test archive integrity | -| `-L n` | Compression level (2–9) | +| `-L n` | Compression level (2--9) | | `-B` | Benchmark all methods on input files | | `-d path` | Extract to specified directory | | `-f` | Overwrite existing files | @@ -98,18 +98,18 @@ UC2/ ## Credits -- **Nico de Vries** — Original UltraCompressor II (1992–1996) -- **Danny Bezemer** — Facilitated source code release (2015) -- **Jan Bobrowski** — Clean-room portable decompressor (unuc2/libunuc2, 2020–2021) -- **Eremey Valetov** — UC2 v3 revival, compression engine, deduplication, and ongoing development +- **Nico de Vries** -- Original UltraCompressor II (1992--1996) +- **Danny Bezemer** -- Facilitated source code release (2015) +- **Jan Bobrowski** -- Clean-room portable decompressor (unuc2/libunuc2, 2020--2021) +- **Eremey Valetov** -- UC2 v3 revival, compression engine, deduplication, and ongoing development See [CREDITS.md](CREDITS.md) for full attribution. ## History -- **v1.0–v2.3** (1992–1996) Original DOS releases by Nico de Vries +- **v1.0--v2.3** (1992--1996) Original DOS releases by Nico de Vries - **2015** Source code released under LGPL-3.0 by Danny Bezemer -- **2020–2021** Jan Bobrowski writes unuc2/libunuc2 (portable decompressor) +- **2020--2021** Jan Bobrowski writes unuc2/libunuc2 (portable decompressor) - **2026** UC2 v3.0.0: cross-platform revival with full compression engine, backward compatibility, and modern deduplication diff --git a/ROADMAP.md b/ROADMAP.md index 84e5ced..09ab985 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -17,7 +17,7 @@ - [x] SuperMaster dictionary support (built-in 49 KB dictionary) - [x] Round-trip testing: 37 unit tests + CLI integration tests - [x] Round-trip testing vs original `uc2pro.exe` in DOSBox - (Direction: original creates → UC2 v3 extracts — verified. + (Direction: original creates -> UC2 v3 extracts -- verified. Reverse direction is a known limitation: the original UC2 Pro cannot read UC2 v3 archives due to compression bitstream differences.) @@ -60,7 +60,7 @@ something no mainstream archiver offers. 6 unit tests. - [x] Delta compression (`uc2_delta.h`): binary diff with COPY/INSERT instructions, hash-based source matching. 96-byte patch in 16KB - file → >50% size savings. 6 unit tests including round-trip. + file -> >50% size savings. 6 unit tests including round-trip. ## Phase 4: Modern Compression Backends @@ -142,17 +142,17 @@ ZK proofs extend the Merkle DAG and encryption layers with privacy-preserving verification. Most valuable for decentralized and compliance scenarios; heavyweight, so implemented as an optional module. -- [ ] **Prove archive integrity without revealing contents** — ZK proof +- [ ] **Prove archive integrity without revealing contents** -- ZK proof that the archive's Merkle root matches claimed file hashes, without exposing the tree structure. Enables auditing of IPFS-shared encrypted archives. -- [ ] **Selective disclosure from encrypted archives** — prove a specific +- [ ] **Selective disclosure from encrypted archives** -- prove a specific file (by hash) exists in an encrypted archive without decrypting anything else. Useful for collaborative encrypted team archives. -- [ ] **Verifiable deduplication** — ZK proof that master-block dedup was +- [ ] **Verifiable deduplication** -- ZK proof that master-block dedup was performed correctly across archives without revealing block contents. Builds trust in distributed dedup without data leaks. -- [ ] **Compliance proofs** — prove properties ("archive created before +- [ ] **Compliance proofs** -- prove properties ("archive created before date Y", "archive does not contain file with hash H") without revealing contents. For regulatory/legal use cases. - [ ] Implementation: Halo2 or Bulletproofs (no trusted setup) via @@ -218,7 +218,7 @@ Bobrowski already shipped prototypes; update for UC2 v3. decompress-on-the-fly with master-block caching) - [ ] Compression tournaments / community challenges - [ ] Neural/learned compression preprocessor (modern platforms only, - not DOS — optional compile-time module) + not DOS -- optional compile-time module) - [ ] Jupyter kernel for interactive archive exploration and compression research (Python, building on foxkernel experience): - Rich HTML tables for archive listings with compression ratios diff --git a/docs/roadmap.rst b/docs/roadmap.rst index 7bbdfd6..90456c0 100644 --- a/docs/roadmap.rst +++ b/docs/roadmap.rst @@ -4,38 +4,38 @@ Roadmap The development roadmap is maintained in ``ROADMAP.md`` at the project root. Key phases: -1. **Decompression MVP** — Done. Portable decompressor, CLI tool, +1. **Decompression MVP** -- Done. Portable decompressor, CLI tool, CMake build system. -2. **Original Compression Engine** — Done. LZ77+Huffman compressor +2. **Original Compression Engine** -- Done. LZ77+Huffman compressor with custom Huffman trees, full backward compatibility, and UC2 personality (``-q`` for scripting). Automated DOSBox-X round-trip validates 4+5 files in both directions. -3. **Modernized Master-Block Deduplication** — Done. +3. **Modernized Master-Block Deduplication** -- Done. CDC with Gear hash, Merkle DAG with content addressing, cross-archive block store, SimHash near-duplicate detection, and delta compression. All Phase 3 items complete. -4. **Modern Compression Backends** — Done. rANS entropy coder (method +4. **Modern Compression Backends** -- Done. rANS entropy coder (method 10), zstd-style dictionary metadata, content-aware preprocessing (BCJ, BWT, delta), LZ4 ultra-fast, benchmark mode (``uc2 -B``). -5. **Quantum-Resistant Encryption** — CRYSTALS-Kyber + AES-256-GCM. +5. **Quantum-Resistant Encryption** -- CRYSTALS-Kyber + AES-256-GCM. -6. **DOS / FreeDOS / Retro-Computing** — DJGPP toolchain, vintage +6. **DOS / FreeDOS / Retro-Computing** -- DJGPP toolchain, vintage hardware support, self-extracting archives. -7. **Cryptographic Integrity & Timestamping** — BLAKE3 hashing done +7. **Cryptographic Integrity & Timestamping** -- BLAKE3 hashing done (pure C, 256-bit digests). Remaining: integration into archive blocks, OpenTimestamps. -8. **Decentralized & Cloud Integration** — IPFS pinning, +8. **Decentralized & Cloud Integration** -- IPFS pinning, content-addressable dedup, cloud archiving. -9. **Zero-Knowledge Proofs** — Privacy-preserving archive verification. +9. **Zero-Knowledge Proofs** -- Privacy-preserving archive verification. -10. **Ecosystem Integrations** — libarchive plugin, streaming dedup +10. **Ecosystem Integrations** -- libarchive plugin, streaming dedup ingestion, file manager plugins. See the full roadmap: `ROADMAP.md on GitHub diff --git a/lib/src/uc2_dict.c b/lib/src/uc2_dict.c index 72fc231..648e08f 100644 --- a/lib/src/uc2_dict.c +++ b/lib/src/uc2_dict.c @@ -1,21 +1,31 @@ -/* Dictionary management for zstd-inspired dictionary compression. - * - * Each dictionary is identified by a 64-bit content hash (FNV-1a) - * and protected by a 32-bit integrity checksum. Dictionaries can - * be serialized to a portable format for storage in block stores - * or as standalone files. */ +/* Dictionary management for zstd-inspired dictionary compression. */ #include "uc2/uc2_dict.h" -#include "uc2/uc2_merkle.h" /* uc2_hash64 */ -#include "uc2/uc2_cdc.h" /* uc2_fnv1a */ +#include "uc2/uc2_merkle.h" +#include "uc2/uc2_cdc.h" #include #include +/* Serialization helpers (little-endian, alignment-safe) */ +static void put32(uint8_t *p, uint32_t v) { + p[0]=v; p[1]=v>>8; p[2]=v>>16; p[3]=v>>24; +} +static void put64(uint8_t *p, uint64_t v) { + for (int i = 0; i < 8; i++) p[i] = (uint8_t)(v >> (i*8)); +} +static uint32_t get32(const uint8_t *p) { + return p[0] | ((uint32_t)p[1]<<8) | ((uint32_t)p[2]<<16) | ((uint32_t)p[3]<<24); +} +static uint64_t get64(const uint8_t *p) { + uint64_t v = 0; + for (int i = 7; i >= 0; i--) v = (v << 8) | p[i]; + return v; +} + int uc2_dict_create(struct uc2_dict *dict, const uint8_t *data, size_t size) { memset(dict, 0, sizeof *dict); if (!data || size == 0) return -1; - dict->data = malloc(size); if (!dict->data) return -1; memcpy(dict->data, data, size); @@ -28,24 +38,23 @@ int uc2_dict_create(struct uc2_dict *dict, const uint8_t *data, size_t size) int uc2_dict_verify(const struct uc2_dict *dict) { if (!dict->data || dict->size == 0) return 0; - uint32_t check = uc2_fnv1a(dict->data, dict->size); - return check == dict->checksum; + return uc2_fnv1a(dict->data, dict->size) == dict->checksum; } +/* Serialized format: magic(4) + id(8) + checksum(4) + size(4) + reserved(4) = 24 bytes */ +#define HDR_SIZE 24 + size_t uc2_dict_serialize(const struct uc2_dict *dict, uint8_t **out) { - size_t total = sizeof(struct uc2_dict_header) + dict->size; + size_t total = HDR_SIZE + dict->size; uint8_t *buf = malloc(total); if (!buf) { *out = NULL; return 0; } - - struct uc2_dict_header *hdr = (struct uc2_dict_header *)buf; - hdr->magic = UC2_DICT_MAGIC; - hdr->id = dict->id; - hdr->checksum = dict->checksum; - hdr->size = dict->size; - hdr->reserved = 0; - memcpy(buf + sizeof *hdr, dict->data, dict->size); - + put32(buf, UC2_DICT_MAGIC); + put64(buf + 4, dict->id); + put32(buf + 12, dict->checksum); + put32(buf + 16, dict->size); + put32(buf + 20, 0); + memcpy(buf + HDR_SIZE, dict->data, dict->size); *out = buf; return total; } @@ -53,18 +62,16 @@ size_t uc2_dict_serialize(const struct uc2_dict *dict, uint8_t **out) int uc2_dict_deserialize(struct uc2_dict *dict, const uint8_t *buf, size_t len) { memset(dict, 0, sizeof *dict); - if (len < sizeof(struct uc2_dict_header)) return -1; - - const struct uc2_dict_header *hdr = (const struct uc2_dict_header *)buf; - if (hdr->magic != UC2_DICT_MAGIC) return -1; - if (sizeof(struct uc2_dict_header) + hdr->size > len) return -1; - - dict->id = hdr->id; - dict->checksum = hdr->checksum; - dict->size = hdr->size; - dict->data = malloc(hdr->size); + if (len < HDR_SIZE) return -1; + if (get32(buf) != UC2_DICT_MAGIC) return -1; + uint32_t size = get32(buf + 16); + if (HDR_SIZE + size > len) return -1; + dict->id = get64(buf + 4); + dict->checksum = get32(buf + 12); + dict->size = size; + dict->data = malloc(size); if (!dict->data) return -1; - memcpy(dict->data, buf + sizeof *hdr, hdr->size); + memcpy(dict->data, buf + HDR_SIZE, size); return 0; }