From 7691dcc4fa583f22d6304f86de87a1027dd955f0 Mon Sep 17 00:00:00 2001 From: Eremey Valetov Date: Thu, 12 Mar 2026 00:52:47 -0400 Subject: [PATCH] Add Sphinx documentation and GitHub Pages deployment Sphinx docs with Furo theme covering CLI usage, library API reference, archive format specification, build instructions, history, and roadmap. GitHub Actions workflow deploys to Pages on push to main. --- .github/workflows/docs.yml | 45 +++++++++++ .gitignore | 3 + docs/_static/.gitkeep | 0 docs/building.rst | 73 +++++++++++++++++ docs/conf.py | 29 +++++++ docs/format.rst | 145 +++++++++++++++++++++++++++++++++ docs/history.rst | 35 ++++++++ docs/index.rst | 22 +++++ docs/library.rst | 159 +++++++++++++++++++++++++++++++++++++ docs/quickstart.rst | 24 ++++++ docs/requirements.txt | 2 + docs/roadmap.rst | 36 +++++++++ docs/usage.rst | 73 +++++++++++++++++ 13 files changed, 646 insertions(+) create mode 100644 .github/workflows/docs.yml create mode 100644 docs/_static/.gitkeep create mode 100644 docs/building.rst create mode 100644 docs/conf.py create mode 100644 docs/format.rst create mode 100644 docs/history.rst create mode 100644 docs/index.rst create mode 100644 docs/library.rst create mode 100644 docs/quickstart.rst create mode 100644 docs/requirements.txt create mode 100644 docs/roadmap.rst create mode 100644 docs/usage.rst diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..d07e0b3 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,45 @@ +name: Docs + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Sphinx + run: pip install -r docs/requirements.txt + + - name: Build docs + run: sphinx-build -b html docs docs/_build/html + + - uses: actions/upload-pages-artifact@v3 + with: + path: docs/_build/html + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index e3f5fc3..1ade37a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ build/ build-*/ cmake-build-*/ +# Docs build +docs/_build/ + # IDE .idea/ .vscode/ diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/building.rst b/docs/building.rst new file mode 100644 index 0000000..350726b --- /dev/null +++ b/docs/building.rst @@ -0,0 +1,73 @@ +Building +======== + +Requirements +------------ + +- CMake >= 3.16 +- C99 compiler: GCC, Clang, or MSVC +- Optional: DJGPP cross-compiler for DOS builds + +Linux / macOS +------------- + +.. code-block:: sh + + cmake -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build + ctest --test-dir build + +The binary is ``build/cli/uc2`` and the library is +``build/lib/libuc2.a``. + +Windows (MSVC) +-------------- + +.. code-block:: sh + + cmake -B build + cmake --build build --config Release + ctest --test-dir build -C Release + +DOS (DJGPP Cross-Compilation) +----------------------------- + +Cross-compile from a Linux host using the DJGPP toolchain: + +.. code-block:: sh + + cmake -B build-dos -DCMAKE_TOOLCHAIN_FILE=cmake/djgpp.cmake + cmake --build build-dos + +This produces a DOS executable suitable for DOSBox or real hardware. + +Build Options +------------- + +.. list-table:: + :header-rows: 1 + :widths: 30 15 55 + + * - Option + - Default + - Description + * - ``UC2_BUILD_TESTS`` + - ``ON`` + - Build test programs + * - ``CMAKE_BUILD_TYPE`` + - (none) + - ``Release``, ``Debug``, ``RelWithDebInfo`` + +Running Tests +------------- + +.. code-block:: sh + + ctest --test-dir build --output-on-failure + +Tests include: + +- **identify**: UC2 magic detection +- **extract**: decompression against reference archives +- **roundtrip**: compress → archive → decompress → verify (8 patterns + × 4 compression levels = 32 tests) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..a534d24 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,29 @@ +project = "UC2" +copyright = "2026, Eremey Valetov" +author = "Eremey Valetov" +release = "3.0.0" + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.githubpages", +] + +templates_path = ["_templates"] +exclude_patterns = ["_build"] + +html_theme = "furo" +html_static_path = ["_static"] +html_title = "UC2 — UltraCompressor II" +html_logo = None +html_favicon = None + +html_theme_options = { + "source_repository": "https://github.com/evvaletov/uc2", + "source_branch": "main", + "source_directory": "docs/", +} + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), +} diff --git a/docs/format.rst b/docs/format.rst new file mode 100644 index 0000000..f741d60 --- /dev/null +++ b/docs/format.rst @@ -0,0 +1,145 @@ +UC2 Archive Format +================== + +This documents the binary format as implemented by the original UC2 +v2.x and supported by UC2 v3. + +Archive Layout +-------------- + +.. code-block:: none + + FHEAD (13 bytes) + XHEAD (16 bytes) + File data blocks (compressed bitstreams) + COMPRESS + compressed central directory + +All multi-byte integers are little-endian. + +FHEAD — File Header +~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :widths: 15 15 70 + + * - Offset + - Size + - Field + * - 0 + - 4 + - Magic: ``UC2\x1A`` (0x1A324355) + * - 4 + - 4 + - Component length + * - 8 + - 4 + - Component length + 0x01B2C3D4 (validation) + * - 12 + - 1 + - Damage protection flag + +XHEAD — Extended Header +~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :widths: 15 15 70 + + * - Offset + - Size + - Field + * - 13 + - 4 + - Cdir volume (always 1) + * - 17 + - 4 + - Cdir offset + * - 21 + - 2 + - Fletcher checksum of raw cdir + * - 23 + - 1 + - Busy flag + * - 24 + - 2 + - Version made by (e.g. 200 = v2.00) + * - 26 + - 2 + - Version needed to extract + * - 28 + - 1 + - Reserved + +Central Directory +----------------- + +The central directory is itself compressed using the UC2 compression +engine. It is located at the offset specified in XHEAD and preceded by +a COMPRESS record. + +Each directory entry begins with a 1-byte type tag: + +.. list-table:: + :widths: 15 85 + + * - 1 + - Directory entry (OSMETA + DIRMETA) + * - 2 + - File entry (OSMETA + FILEMETA + COMPRESS + LOCATION) + * - 3 + - Master entry (MASMETA + COMPRESS + LOCATION) + * - 4 + - End of central directory + +The directory ends with XTAIL (17 bytes) + archive serial (4 bytes). + +Compression Format +------------------ + +UC2 uses LZ77 with Huffman entropy coding. The bitstream consists of +blocks, each containing: + +1. **Block-present flag** (1 bit): 1 = block follows, 0 = end of stream. + +2. **Huffman tree** encoded as: + + - Tree-changed flag (1 bit): 0 = use default tree, 1 = new tree. + - Type flags (2 bits): ``has_lo | has_hi << 1``, controlling which + symbol ranges are encoded. + - Tree-encoding tree (15 × 3-bit lengths). + - Delta-coded symbol lengths with RLE (344 symbols total = + 256 literals + 60 distance + 28 length). + +3. **Compressed data**: Huffman-coded literals and distance/length pairs. + +4. **End-of-block marker**: distance = 64001 with length = 3. + +Distance Encoding +~~~~~~~~~~~~~~~~~ + +60 distance symbols in 4 tiers: + +- Tier 0: distances 1--15 (0 extra bits) +- Tier 1: distances 16--255 (4 extra bits) +- Tier 2: distances 256--4095 (8 extra bits) +- Tier 3: distances 4096--64000 (12 extra bits) + +Length Encoding +~~~~~~~~~~~~~~~ + +28 length symbols with varying extra bits, covering lengths 3--35482. + +Delta-Coded Trees +~~~~~~~~~~~~~~~~~ + +Symbol code lengths are delta-coded against the previous block's +lengths using the ``vval`` lookup table. The first block's default +lengths are hard-coded. The delta stream uses 14 delta codes (0--13) +plus a repeat code for RLE compression. + +Fletcher Checksum +----------------- + +UC2 uses an XOR-based Fletcher checksum (initial value 0xA55A) for +both file data integrity and central directory validation. Bytes are +processed in little-endian 16-bit words with a carry flag for +odd-length data. diff --git a/docs/history.rst b/docs/history.rst new file mode 100644 index 0000000..dcd2559 --- /dev/null +++ b/docs/history.rst @@ -0,0 +1,35 @@ +History +======= + +.. list-table:: + :widths: 20 80 + + * - **1992--1996** + - UltraCompressor II created by Nico de Vries. Releases v1.0 + through v2.3 for DOS. Notable features: LZ77+Huffman + compression, master-block deduplication, file versioning, + and multi-volume spanning. + + * - **2015** + - Source code released under LGPL-3.0 by Danny Bezemer. + + * - **2020--2021** + - Jan Bobrowski writes `unuc2/libunuc2 + `_, a clean-room portable + decompressor in C. + + * - **2026** + - UC2 v3.0.0: cross-platform revival by Eremey Valetov. + CMake build system, Linux/macOS/Windows/DOS targets, + original compression engine reimplemented, Sphinx + documentation, CI/CD pipeline. + +Credits +------- + +- **Nico de Vries** — original UC2 author +- **Danny Bezemer** — source code release +- **Jan Bobrowski** — portable decompressor (libunuc2) +- **Jan-Pieter Cornet** — testing and format documentation +- **Vladislav Sagunov** — UC2 resources and documentation +- **Eremey Valetov** — v3.0.0 revival and ongoing development diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..4335f0f --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,22 @@ +UC2 — UltraCompressor II +======================== + +A cross-platform revival of UltraCompressor II, the DOS-era archiver by +Nico de Vries (1992--1996). UC2 was notable for its advanced +deduplication ("master blocks"), file versioning, and competitive +compression ratios on the hardware of its day. + +UC2 v3 brings it back as a modern, portable C99 tool with both +decompression and compression, targeting Linux, macOS, Windows, and DOS. + +.. toctree:: + :maxdepth: 2 + :caption: Contents + + quickstart + usage + library + format + building + history + roadmap diff --git a/docs/library.rst b/docs/library.rst new file mode 100644 index 0000000..eca404a --- /dev/null +++ b/docs/library.rst @@ -0,0 +1,159 @@ +Library API (libuc2) +==================== + +``libuc2`` provides C99 functions for reading, extracting, and +compressing UC2 archives. The library is callback-based: callers supply +I/O and memory callbacks, making it suitable for embedded, DOS, and +freestanding environments. + +Header: ```` + +Archive Reading +--------------- + +.. c:function:: int uc2_identify(void *magic, unsigned magic_size) + + Check whether a buffer contains a UC2 archive header. + + :param magic: Pointer to the first 4--21 bytes of the file. + :param magic_size: Number of bytes available. + :returns: ``1`` if UC2, ``0`` if not, ``-1`` if more bytes needed. + +.. c:function:: uc2_handle uc2_open(struct uc2_io *io, void *io_ctx) + + Open a UC2 archive. The caller provides I/O callbacks via + :c:type:`uc2_io`. Returns ``NULL`` on allocation failure. + +.. c:function:: uc2_handle uc2_close(uc2_handle h) + + Close the archive and free all resources. Always returns ``NULL``. + +Directory Enumeration +--------------------- + +.. c:function:: int uc2_read_cdir(uc2_handle h, struct uc2_entry *entry) + + Read the next central directory entry. + + :returns: + - ``UC2_End`` (0): end of directory, *entry* not filled. + - ``UC2_BareEntry`` (1): entry filled, no tags. + - ``UC2_TaggedEntry`` (3): entry filled, call :c:func:`uc2_get_tag` + to read tags (long filename, etc.). + - Negative value on error. + + Directories appear before their contents. Duplicate filenames are + listed oldest-first. + +.. c:function:: int uc2_get_tag(uc2_handle h, struct uc2_entry *entry, char **tag, void **data, unsigned *data_len) + + Read a tag from a tagged entry. Call repeatedly until it returns + ``UC2_End``. + +.. c:function:: int uc2_finish_cdir(uc2_handle h, char label[12]) + + Read the archive tail and retrieve the volume label. + +Extraction +---------- + +.. c:function:: int uc2_extract(uc2_handle h, struct uc2_xinfo *xi, unsigned size, int (*write)(void *ctx, const void *ptr, unsigned len), void *ctx) + + Decompress a file entry. Call only after the entire central + directory has been read. The *write* callback receives decompressed + data in chunks. + +Compression +----------- + +.. c:function:: int uc2_compress(int level, int (*read)(void *ctx, void *buf, unsigned len), void *read_ctx, int (*write)(void *ctx, const void *ptr, unsigned len), void *write_ctx, unsigned size, unsigned short *checksum_out, unsigned *compressed_size_out) + + Compress raw data into a UC2 bitstream (no archive framing). + + :param level: Compression level: 2 = Fast, 3 = Normal, 4 = Tight + (default), 5 = Ultra. + :param read: Callback returning bytes read (0 at EOF, <0 on error). + :param write: Callback returning <0 on error. + :param size: Total input size in bytes. + :param checksum_out: Receives the Fletcher checksum of the input. + :param compressed_size_out: Receives the compressed size. + :returns: 0 on success, negative ``UC2_*`` error code on failure. + +I/O Callbacks +------------- + +.. c:struct:: uc2_io + + .. c:member:: int (*read)(void *io_ctx, unsigned pos, void *buf, unsigned len) + + Read *len* bytes from the archive at offset *pos* into *buf*. + Return number of bytes read (less if EOF), or negative on error. + + .. c:member:: void *(*alloc)(void *io_ctx, unsigned size) + + Allocate memory. Return ``NULL`` on failure. + + .. c:member:: void (*free)(void *io_ctx, void *ptr) + + Free memory. + + .. c:member:: void (*warn)(void *io_ctx, char *fmt, ...) + + Optional warning callback. + +Data Structures +--------------- + +.. c:struct:: uc2_entry + + A directory entry. + + .. c:member:: unsigned dirid + + Parent directory (0 = root). + + .. c:member:: unsigned id + + Directory index (directories only). + + .. c:member:: unsigned size + + Uncompressed file size. + + .. c:member:: unsigned csize + + Compressed file size. + + .. c:member:: unsigned dos_time + + DOS-format timestamp. + + .. c:member:: unsigned char attr + + DOS file attributes. + + .. c:member:: char name[300] + + Filename (UTF-8, NUL-terminated). Populated after tags are read. + +Error Codes +----------- + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Constant + - Meaning + * - ``UC2_UserFault`` (-2) + - User callback refused to cooperate + * - ``UC2_BadState`` (-3) + - API called in wrong order + * - ``UC2_Damaged`` (-4) + - Archive data is corrupt + * - ``UC2_Truncated`` (-5) + - Unexpected end of data + * - ``UC2_Unimplemented`` (-6) + - Feature not yet implemented + * - ``UC2_InternalError`` (-7) + - Internal logic error diff --git a/docs/quickstart.rst b/docs/quickstart.rst new file mode 100644 index 0000000..affca11 --- /dev/null +++ b/docs/quickstart.rst @@ -0,0 +1,24 @@ +Quick Start +=========== + +Building +-------- + +Requires CMake >= 3.16 and a C99 compiler (GCC, Clang, or MSVC). + +.. code-block:: sh + + cmake -B build + cmake --build build + +The binary is at ``build/cli/uc2``. + +Basic Usage +----------- + +.. code-block:: sh + + uc2 archive.uc2 # Extract all files + uc2 -l archive.uc2 # List contents + uc2 -t archive.uc2 # Test archive integrity + uc2 -d /tmp/out archive.uc2 # Extract to directory diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..a4213e9 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx>=7.0 +furo diff --git a/docs/roadmap.rst b/docs/roadmap.rst new file mode 100644 index 0000000..03f4b7a --- /dev/null +++ b/docs/roadmap.rst @@ -0,0 +1,36 @@ +Roadmap +======= + +The development roadmap is maintained in ``ROADMAP.md`` at the project +root. Key phases: + +1. **Decompression MVP** — Done. Portable decompressor, CLI tool, + CMake build system. + +2. **Original Compression Engine** — Done. LZ77+Huffman compressor + producing bitstreams compatible with the original format. + +3. **Modernized Master-Block Deduplication** — Content-defined + chunking, Merkle DAG, cross-archive dedup, near-duplicate detection. + +4. **Modern Compression Backends** — ANS entropy coding, + zstd-inspired dictionary compression, content-aware preprocessing. + +5. **Quantum-Resistant Encryption** — CRYSTALS-Kyber + AES-256-GCM. + +6. **DOS / FreeDOS / Retro-Computing** — DJGPP toolchain, vintage + hardware support, self-extracting archives. + +7. **Cryptographic Integrity & Timestamping** — BLAKE3 hashing, + OpenTimestamps. + +8. **Decentralized & Cloud Integration** — IPFS pinning, + content-addressable dedup, cloud archiving. + +9. **Zero-Knowledge Proofs** — Privacy-preserving archive verification. + +10. **Ecosystem Integrations** — libarchive plugin, streaming dedup + ingestion, file manager plugins. + +See the full roadmap: `ROADMAP.md on GitHub +`_. diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 0000000..a342395 --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,73 @@ +Command-Line Usage +================== + +Synopsis +-------- + +.. code-block:: none + + uc2 [options] archive.uc2 [patterns...] + +Modes +----- + +``uc2 archive.uc2`` + Extract all files to the current directory. + +``uc2 -l archive.uc2`` + List archive contents. + +``uc2 -t archive.uc2`` + Test archive integrity (decompress and verify checksums without + writing files). + +``uc2 -p archive.uc2 filename`` + Extract a file to stdout. + +Options +------- + +.. list-table:: + :header-rows: 1 + :widths: 15 85 + + * - Flag + - Description + * - ``-l`` + - List archive contents + * - ``-t`` + - Test archive integrity + * - ``-a`` + - Include all file versions (not just latest) + * - ``-d path`` + - Extract to specified directory + * - ``-f`` + - Overwrite existing files + * - ``-p`` + - Extract to stdout + * - ``-D`` + - Skip directory metadata; ``-DD`` also skips file metadata + * - ``-T`` + - Tab-separated output (for scripting) + +Pattern Matching +---------------- + +File patterns use glob syntax. Only files matching the pattern are +listed or extracted: + +.. code-block:: sh + + uc2 -l archive.uc2 '*.txt' # List only .txt files + uc2 archive.uc2 'src/*' # Extract src/ subtree + +Exit Codes +---------- + +.. list-table:: + :widths: 15 85 + + * - ``0`` + - Success + * - ``1`` + - Error (damaged archive, I/O failure, etc.)