diff options
Diffstat (limited to 'Src/external_dependencies/openmpt-trunk/include/ancient')
152 files changed, 15914 insertions, 0 deletions
diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/LICENSE b/Src/external_dependencies/openmpt-trunk/include/ancient/LICENSE new file mode 100644 index 00000000..d7a1d610 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/LICENSE @@ -0,0 +1,25 @@ +BSD 2-Clause License + +Copyright (c) 2017-2022, Teemu Suutari +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/OpenMPT.txt b/Src/external_dependencies/openmpt-trunk/include/ancient/OpenMPT.txt new file mode 100644 index 00000000..b52f9edf --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/OpenMPT.txt @@ -0,0 +1,2 @@ + * ancient <https://github.com/temisu/ancient> v2.0.0 (2022-01-12)
+ * Unnecssary files have been removed.
diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/api/ancient/ancient.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/api/ancient/ancient.hpp new file mode 100644 index 00000000..cdef5034 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/api/ancient/ancient.hpp @@ -0,0 +1,148 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef ANCIENT_ANCIENT_HPP +#define ANCIENT_ANCIENT_HPP + +#ifndef ANCIENT_API +#if defined(ANCIENT_API_DECLSPEC_DLLEXPORT) +#define ANCIENT_API __declspec(dllexport) +#endif +#if defined(ANCIENT_API_DECLSPEC_DLLIMPORT) +#define ANCIENT_API __declspec(dllimport) +#endif +#if defined(ANCIENT_API_VISIBILITY_DEFAULT_USED) +#define ANCIENT_API __attribute__((visibility("default"))) __attribute__((used)) +#endif +#if defined(ANCIENT_API_VISIBILITY_DEFAULT) +#define ANCIENT_API __attribute__((visibility("default"))) +#endif +#if defined(ANCIENT_API_VISIBILITY_HIDDEN) +#define ANCIENT_API __attribute__((visibility("hidden"))) +#endif +#ifndef ANCIENT_API +#define ANCIENT_API +#endif +#endif + +#include <exception> +#include <memory> +#include <optional> +#include <string> +#include <vector> + +#include <cstddef> +#include <cstdint> + +namespace ancient +{ + +namespace internal +{ +namespace APIv2 +{ +class DecompressorImpl; +} +} + +inline namespace APIv2 +{ + +// just a base class to easily catch all the errors +class ANCIENT_API Error : public std::exception +{ +public: + Error() noexcept; + virtual ~Error(); +}; + +class ANCIENT_API InvalidFormatError : public Error +{ +public: + InvalidFormatError() noexcept; + virtual ~InvalidFormatError(); +}; + +class ANCIENT_API DecompressionError : public Error +{ +public: + DecompressionError() noexcept; + virtual ~DecompressionError(); +}; + +class ANCIENT_API VerificationError : public Error +{ +public: + VerificationError() noexcept; + virtual ~VerificationError(); +}; + +class ANCIENT_API Decompressor final +{ + +public: + + // Detect signature whether it matches to any known compressor + // This does not guarantee the data is decompressable though, only signature is read + static bool detect(const std::vector<uint8_t> &packedData) noexcept; + static bool detect(const uint8_t *packedData,size_t packedSize) noexcept; + + // Main entrypoint + // if verify=true then check the packedData for errors: CRC or other checksum if available + // check exactSizeKnown from size documentation + // can throw InvalidFormatError if stream is not recognized or it is invalid + // can throw VerificationError if verify enabled and checksum does not match + explicit Decompressor(const std::vector<uint8_t> &packedData,bool exactSizeKnown,bool verify); + explicit Decompressor(const uint8_t *packedData,size_t packedSize,bool exactSizeKnown,bool verify); + + // Name returned is human readable long name + const std::string &getName() const noexcept; + + // the functions are there to protect against "accidental" large files when parsing headers + // a.k.a. 16M should be enough for everybody (sizes do not have to accurate i.e. + // compressors can exclude header content for simplification) + // This entirely ok for the context of "old computers" and their files, + // for other usages these need to be tuned up + static size_t getMaxPackedSize() noexcept; + static size_t getMaxRawSize() noexcept; + + // PackedSize or RawSize are taken from the stream if available, std::nullopt otherwise. + // for those compressors having no known sizes, running decompression will update + // the values. + // There are exceptions: Some decompressors need to exact size of the packed data + // in order to decompress. For those providing a indefinitely size packed stream + // will not work + // use the "exactSizeKnown" flag for create to tell whether you know the size or not + std::optional<size_t> getPackedSize() const noexcept; + std::optional<size_t> getRawSize() const noexcept; + + // in case of disk image based formats the data does not necessarily start + // from logical beginnig of the image but it is offsetted inside the logical image (f.e. DMS). + // getDataOffset will return the offset which can also be 0, or std::nullopt if not image-based format. + // getImageSize will return the size of the the logical image, or std::nullopt if not image-based format. + std::optional<size_t> getImageSize() const noexcept; + std::optional<size_t> getImageOffset() const noexcept; + + // Actual decompression. + // verify checksum if verify==true + // can throw DecompressionError if stream cant be unpacked + // can throw VerificationError if verify enabled and checksum does not match + std::vector<uint8_t> decompress(bool verify); + + ~Decompressor(); + +private: + + std::unique_ptr<internal::APIv2::DecompressorImpl> m_impl; + +private: + + Decompressor(const Decompressor&)=delete; + Decompressor& operator=(const Decompressor&)=delete; + +}; + +} + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/ACCADecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ACCADecompressor.cpp new file mode 100644 index 00000000..5e815804 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ACCADecompressor.cpp @@ -0,0 +1,112 @@ +/* Copyright (C) Teemu Suutari */ + +#include "ACCADecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool ACCADecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("ACCA"); +} + +std::shared_ptr<XPKDecompressor> ACCADecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<ACCADecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +ACCADecompressor::ACCADecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +ACCADecompressor::~ACCADecompressor() +{ + // nothing needed +} + +const std::string &ACCADecompressor::getSubName() const noexcept +{ + static std::string name="XPK-ACCA: Andre's code compression algorithm"; + return name; +} + +void ACCADecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBit=[&]()->uint32_t + { + return bitReader.readBitsBE16(1); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + while (!outputStream.eof()) + { + if (!readBit()) + { + outputStream.writeByte(readByte()); + } else { + static const uint8_t staticBytes[16]={ + 0x00,0x01,0x02,0x03,0x04,0x08,0x10,0x20, + 0x40,0x55,0x60,0x80,0xaa,0xc0,0xe0,0xff}; + + uint8_t tmp=readByte(); + uint32_t count=tmp&0xf; + uint32_t code=tmp>>4; + uint32_t distance=0; + uint8_t repeatChar=0; + bool doRLE=false; + switch (code) + { + case 0: + repeatChar=readByte(); + case 14: + count+=3; + doRLE=true; + break; + + case 1: + count=(count|(uint32_t(readByte())<<4))+19; + repeatChar=readByte(); + doRLE=true; + break; + + case 2: + repeatChar=staticBytes[count]; + count=2; + doRLE=true; + break; + + case 15: + distance=(count|(uint32_t(readByte())<<4))+3; + count=uint32_t(readByte())+14; + break; + + default: /* 3 to 13 */ + distance=(count|(uint32_t(readByte())<<4))+3; + count=code; + break; + } + if (doRLE) + { + for (uint32_t i=0;i<count;i++) outputStream.writeByte(repeatChar); + } else { + outputStream.copy(distance,count); + } + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/ACCADecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ACCADecompressor.hpp new file mode 100644 index 00000000..1fd40e15 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ACCADecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef ACCADECOMPRESSOR_HPP +#define ACCADECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class ACCADecompressor : public XPKDecompressor +{ +public: + ACCADecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~ACCADecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/API.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/API.cpp new file mode 100644 index 00000000..996e0f32 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/API.cpp @@ -0,0 +1,189 @@ +/* Copyright (C) Teemu Suutari */ + +#include "ancient.hpp" +#include "Decompressor.hpp" +#include "common/Buffer.hpp" +#include "common/StaticBuffer.hpp" +#include "common/WrappedVectorBuffer.hpp" + +#include <memory> + +namespace ancient +{ + +namespace internal +{ + +namespace APIv2 +{ + +class DecompressorImpl +{ +public: + ConstStaticBuffer _buffer; + std::shared_ptr<Decompressor> _decompressor; +public: + DecompressorImpl(const std::vector<uint8_t> &packedData,bool exactSizeKnown,bool verify) : + _buffer(packedData.data(), packedData.size()), + _decompressor(Decompressor::create(_buffer, exactSizeKnown, verify)) + { + return; + } + DecompressorImpl(const uint8_t *packedData,size_t packedSize,bool exactSizeKnown,bool verify) : + _buffer(packedData, packedSize), + _decompressor(Decompressor::create(_buffer, exactSizeKnown, verify)) + { + return; + } +}; + +} + +} + +inline namespace APIv2 +{ + +Error::Error() noexcept +{ + // nothing needed +} + +Error::~Error() +{ + // nothing needed +} + +InvalidFormatError::InvalidFormatError() noexcept +{ + // nothing needed +} + +InvalidFormatError::~InvalidFormatError() +{ + // nothing needed +} + +DecompressionError::DecompressionError() noexcept +{ + // nothing needed +} + +DecompressionError::~DecompressionError() +{ + // nothing needed +} + +VerificationError::VerificationError() noexcept +{ + // nothing needed +} + +VerificationError::~VerificationError() +{ + // nothing needed +} + +// --- + +bool Decompressor::detect(const std::vector<uint8_t> &packedData) noexcept +{ + return internal::Decompressor::detect(internal::ConstStaticBuffer(packedData.data(), packedData.size())); +} + +bool Decompressor::detect(const uint8_t *packedData, size_t packedSize) noexcept +{ + return internal::Decompressor::detect(internal::ConstStaticBuffer(packedData, packedSize)); +} + +Decompressor::Decompressor(const std::vector<uint8_t> &packedData,bool exactSizeKnown,bool verify) : + m_impl(std::make_unique<internal::APIv2::DecompressorImpl>(packedData, exactSizeKnown, verify)) +{ + return; +} + +Decompressor::Decompressor(const uint8_t *packedData,size_t packedSize,bool exactSizeKnown,bool verify) : + m_impl(std::make_unique<internal::APIv2::DecompressorImpl>(packedData, packedSize, exactSizeKnown, verify)) +{ + return; +} + +const std::string &Decompressor::getName() const noexcept +{ + return m_impl->_decompressor->getName(); +} + +size_t Decompressor::getMaxPackedSize() noexcept +{ + return internal::Decompressor::getMaxPackedSize(); +} + +size_t Decompressor::getMaxRawSize() noexcept +{ + return internal::Decompressor::getMaxRawSize(); +} + +std::optional<size_t> Decompressor::getPackedSize() const noexcept +{ + size_t packedSize=m_impl->_decompressor->getPackedSize(); + if (packedSize==0) + { + return std::nullopt; + } + return packedSize; +} + +std::optional<size_t> Decompressor::getRawSize() const noexcept +{ + size_t rawSize=m_impl->_decompressor->getRawSize(); + if (rawSize==0) + { + return std::nullopt; + } + return rawSize; +} + +std::optional<size_t> Decompressor::getImageSize() const noexcept +{ + size_t imageSize=m_impl->_decompressor->getImageSize(); + size_t imageOffset=m_impl->_decompressor->getImageOffset(); + bool isImage=((imageSize>0)||(imageOffset>0)); + if (!isImage) + { + return std::nullopt; + } + return imageSize; +} + +std::optional<size_t> Decompressor::getImageOffset() const noexcept +{ + size_t imageSize=m_impl->_decompressor->getImageSize(); + size_t imageOffset=m_impl->_decompressor->getImageOffset(); + bool isImage=((imageSize>0)||(imageOffset>0)); + if (!isImage) + { + return std::nullopt; + } + return imageOffset; +} + +std::vector<uint8_t> Decompressor::decompress(bool verify) +{ + std::vector<uint8_t> result((m_impl->_decompressor->getRawSize())?m_impl->_decompressor->getRawSize():m_impl->_decompressor->getMaxRawSize()); + { + internal::WrappedVectorBuffer buffer(result); + m_impl->_decompressor->decompress(buffer, verify); + } + result.resize(m_impl->_decompressor->getRawSize()); + result.shrink_to_fit(); + return result; +} + +Decompressor::~Decompressor() +{ + // nothing needed +} + +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/ARTMDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ARTMDecompressor.cpp new file mode 100644 index 00000000..6a84e7cc --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ARTMDecompressor.cpp @@ -0,0 +1,122 @@ +/* Copyright (C) Teemu Suutari */ + +#include "ARTMDecompressor.hpp" +#include "RangeDecoder.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool ARTMDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("ARTM"); +} + +std::shared_ptr<XPKDecompressor> ARTMDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<ARTMDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +ARTMDecompressor::ARTMDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); + if (packedData.size()<2) throw Decompressor::InvalidFormatError(); +} + +ARTMDecompressor::~ARTMDecompressor() +{ + // nothing needed +} + +const std::string &ARTMDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-ARTM: Arithmetic encoding compressor"; + return name; +} + +// There really is not much to see here. +// Except maybe for the fact that there is extra symbol defined but never used. +// It is used in the original implementation as a terminator. In here it is considered as an error. +// Logically it would decode into null-character (practically it would be instant buffer overflow) +void ARTMDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + class BitReader : public RangeDecoder::BitReader + { + public: + BitReader(ForwardInputStream &stream) : + _reader(stream) + { + // nothing needed + } + + virtual ~BitReader() + { + // nothing needed + } + + virtual uint32_t readBit() override final + { + return _reader.readBits8(1); + } + + private: + LSBBitReader<ForwardInputStream> _reader; + }; + + + ForwardInputStream inputStream(_packedData,0,_packedData.size(),true); + ForwardOutputStream outputStream(rawData,0,rawData.size()); + BitReader bitReader(inputStream); + + uint16_t initialValue=0; + for (uint32_t i=0;i<16;i++) initialValue=(initialValue<<1)|bitReader.readBit(); + RangeDecoder decoder(bitReader,initialValue); + + uint8_t characters[256]; + uint16_t frequencies[256]; + uint16_t frequencySums[256]; + + for (uint32_t i=0;i<256;i++) + { + characters[i]=i; + frequencies[i]=1; + frequencySums[i]=256-i; + } + uint16_t frequencyTotal=257; + + while (!outputStream.eof()) + { + uint16_t value=decoder.decode(frequencyTotal); + uint16_t symbol; + for (symbol=0;symbol<256;symbol++) + if (frequencySums[symbol]<=value) break; + if (symbol==256) throw Decompressor::DecompressionError(); + decoder.scale(frequencySums[symbol],frequencySums[symbol]+frequencies[symbol],frequencyTotal); + outputStream.writeByte(characters[symbol]); + + if (frequencyTotal==0x3fffU) + { + frequencyTotal=1; + for (int32_t i=255;i>=0;i--) + { + frequencySums[i]=frequencyTotal; + frequencyTotal+=frequencies[i]=(frequencies[i]+1)>>1; + } + } + + uint16_t i; + for (i=symbol;i&&frequencies[i-1]==frequencies[i];i--); + if (i!=symbol) + std::swap(characters[symbol],characters[i]); + frequencies[i]++; + while (i--) frequencySums[i]++; + frequencyTotal++; + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/ARTMDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ARTMDecompressor.hpp new file mode 100644 index 00000000..7a9c6b5a --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ARTMDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef ARTMDECOMPRESSOR_HPP +#define ARTMDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class ARTMDecompressor : public XPKDecompressor +{ +public: + ARTMDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~ARTMDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/BLZWDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/BLZWDecompressor.cpp new file mode 100644 index 00000000..7cc35e9f --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/BLZWDecompressor.cpp @@ -0,0 +1,127 @@ +/* Copyright (C) Teemu Suutari */ + +#include "BLZWDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool BLZWDecompressor::detectHeaderXPK(uint32_t hdr) +{ + return hdr==FourCC("BLZW"); +} + +std::shared_ptr<XPKDecompressor> BLZWDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<BLZWDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +BLZWDecompressor::BLZWDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); + _maxBits=_packedData.readBE16(0); + if (_maxBits<9 || _maxBits>20) throw Decompressor::InvalidFormatError();; + _stackLength=uint32_t(_packedData.readBE16(2))+5; +} + +BLZWDecompressor::~BLZWDecompressor() +{ + // nothing needed +} + +const std::string &BLZWDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-BLZW: LZW-compressor"; + return name; +} + +void BLZWDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,4,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + uint32_t maxCode=1<<_maxBits; + auto prefix=std::make_unique<uint32_t[]>(maxCode-259); + auto suffix=std::make_unique<uint8_t[]>(maxCode-259); + auto stack=std::make_unique<uint8_t[]>(_stackLength); + + uint32_t freeIndex,codeBits,prevCode,newCode; + + auto suffixLookup=[&](uint32_t code)->uint32_t + { + if (code>=freeIndex) throw Decompressor::DecompressionError(); + return (code<259)?code:suffix[code-259]; + }; + + auto insert=[&](uint32_t code) + { + uint32_t stackPos=0; + newCode=suffixLookup(code); + while (code>=259) + { + if (stackPos+1>=_stackLength) throw Decompressor::DecompressionError(); + stack[stackPos++]=newCode; + code=prefix[code-259]; + newCode=suffixLookup(code); + } + stack[stackPos++]=newCode; + while (stackPos) outputStream.writeByte(stack[--stackPos]); + }; + + auto init=[&]() + { + codeBits=9; + freeIndex=259; + prevCode=readBits(codeBits); + insert(prevCode); + }; + + init(); + while (!outputStream.eof()) + { + uint32_t code=readBits(codeBits); + switch (code) + { + case 256: + throw Decompressor::DecompressionError(); + break; + + case 257: + init(); + break; + + case 258: + codeBits++; + break; + + default: + if (code>=freeIndex) + { + uint32_t tmp=newCode; + insert(prevCode); + outputStream.writeByte(tmp); + } else insert(code); + if (freeIndex<maxCode) + { + suffix[freeIndex-259]=newCode; + prefix[freeIndex-259]=prevCode; + freeIndex++; + } + prevCode=code; + break; + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/BLZWDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/BLZWDecompressor.hpp new file mode 100644 index 00000000..9080784c --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/BLZWDecompressor.hpp @@ -0,0 +1,34 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef BLZWDECOMPRESSOR_HPP +#define BLZWDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class BLZWDecompressor : public XPKDecompressor +{ +public: + BLZWDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~BLZWDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr); + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _maxBits=0; + size_t _stackLength=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/BZIP2Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/BZIP2Decompressor.cpp new file mode 100644 index 00000000..65a24aef --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/BZIP2Decompressor.cpp @@ -0,0 +1,399 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstdint> +#include <cstring> + +#include "BZIP2Decompressor.hpp" +#include "HuffmanDecoder.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/MemoryBuffer.hpp" +#include "common/CRC32.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool BZIP2Decompressor::detectHeader(uint32_t hdr) noexcept +{ + return ((hdr&0xffff'ff00U)==FourCC("BZh\0") && (hdr&0xffU)>='1' && (hdr&0xffU)<='9'); +} + +bool BZIP2Decompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return (hdr==FourCC("BZP2")); +} + +std::shared_ptr<Decompressor> BZIP2Decompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) +{ + return std::make_shared<BZIP2Decompressor>(packedData,exactSizeKnown,verify); +} + +std::shared_ptr<XPKDecompressor> BZIP2Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<BZIP2Decompressor>(hdr,recursionLevel,packedData,state,verify); +} + +BZIP2Decompressor::BZIP2Decompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : + _packedData(packedData), + _packedSize(0) +{ + uint32_t hdr=packedData.readBE32(0); + if (!detectHeader(hdr)) throw Decompressor::InvalidFormatError();; + _blockSize=((hdr&0xffU)-'0')*100'000; +} + +BZIP2Decompressor::BZIP2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData), + _packedSize(_packedData.size()) +{ + uint32_t blockHdr=packedData.readBE32(0); + if (!detectHeader(blockHdr)) throw Decompressor::InvalidFormatError();; + _blockSize=((blockHdr&0xffU)-'0')*100'000; +} + +BZIP2Decompressor::~BZIP2Decompressor() +{ + // nothing needed +} + +const std::string &BZIP2Decompressor::getName() const noexcept +{ + static std::string name="bz2: bzip2"; + return name; +} + +const std::string &BZIP2Decompressor::getSubName() const noexcept +{ + static std::string name="XPK-BZP2: bzip2"; + return name; +} + +size_t BZIP2Decompressor::getPackedSize() const noexcept +{ + // no way to know before decompressing + return _packedSize; +} + + +size_t BZIP2Decompressor::getRawSize() const noexcept +{ + // same thing, decompression needed first + return _rawSize; +} + +void BZIP2Decompressor::decompressImpl(Buffer &rawData,bool verify) +{ + size_t packedSize=_packedSize?_packedSize:_packedData.size(); + size_t rawSize=_rawSize?_rawSize:rawData.size(); + + ForwardInputStream inputStream(_packedData,4,packedSize); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawSize); + + // stream verification + // + // there is so much wrong in bzip2 CRC-calculation :( + // 1. The bit ordering is opposite what everyone else does with CRC32 + // 2. The block CRCs are calculated separately, no way of calculating a complete + // CRC without knowing the block layout + // 3. The CRC is the end of the stream and the stream is bit aligned. You + // can't read CRC without decompressing the stream. + uint32_t crc=0; + auto calculateBlockCRC=[&](size_t blockPos,size_t blockSize) + { + crc=(crc<<1)|(crc>>31); + crc^=CRC32Rev(rawData,blockPos,blockSize,0); + }; + + HuffmanDecoder<uint8_t> selectorDecoder + { + // incomplete Huffman table. errors possible + HuffmanCode<uint8_t>{1,0b000000,0}, + HuffmanCode<uint8_t>{2,0b000010,1}, + HuffmanCode<uint8_t>{3,0b000110,2}, + HuffmanCode<uint8_t>{4,0b001110,3}, + HuffmanCode<uint8_t>{5,0b011110,4}, + HuffmanCode<uint8_t>{6,0b111110,5} + }; + + HuffmanDecoder<int32_t> deltaDecoder + { + HuffmanCode<int32_t>{1,0b00,0}, + HuffmanCode<int32_t>{2,0b10,1}, + HuffmanCode<int32_t>{2,0b11,-1} + }; + + MemoryBuffer tmpBuffer(_blockSize); + uint8_t *tmpBufferPtr=tmpBuffer.data(); + + // This is the dark, ancient secret of bzip2. + // versions before 0.9.5 had a data randomization for "too regular" + // data problematic for the bwt-implementation at that time. + // although it is never utilized anymore, the support is still there + // And this is exactly the kind of ancient stuff we want to support :) + // + // On this specific part (since it is a table of magic numbers) + // we have no way other than copying it from the original reference + +// Table has a separate copyright, lets have it as a separate file as well +#include "BZIP2Table.hpp" + + for (;;) + { + uint32_t blockHdrHigh=readBits(32); + uint32_t blockHdrLow=readBits(16); + if (blockHdrHigh==0x31415926U && blockHdrLow==0x5359U) + { + // a block + + // this is rather spaghetti... + readBits(32); // block crc, not interested + bool randomized=readBit(); + + // basically the random inserted is one LSB after n-th bytes + // per defined in the table. + uint32_t randomPos=1; + uint32_t randomCounter=randomTable[0]-1; + auto randomBit=[&]()->bool + { + // Beauty is in the eye of the beholder: this is smallest form to hide the ugliness + return (!randomCounter--)?randomCounter=randomTable[randomPos++&511]:false; + }; + + uint32_t currentPtr=readBits(24); + + uint32_t currentBlockSize=0; + { + uint32_t numHuffmanItems=2; + uint32_t huffmanValues[256]; + + { + // this is just a little bit inefficient but still we reading bit by bit since + // reference does it. (bitsream format details do not spill over) + std::vector<bool> usedMap(16); + for (uint32_t i=0;i<16;i++) usedMap[i]=readBit(); + + std::vector<bool> huffmanMap(256); + for (uint32_t i=0;i<16;i++) + { + for (uint32_t j=0;j<16;j++) + huffmanMap[i*16+j]=(usedMap[i])?readBit():false; + } + + for (uint32_t i=0;i<256;i++) if (huffmanMap[i]) numHuffmanItems++; + if (numHuffmanItems==2) throw DecompressionError(); + + for (uint32_t currentValue=0,i=0;i<256;i++) + if (huffmanMap[i]) huffmanValues[currentValue++]=i; + } + + uint32_t huffmanGroups=readBits(3); + if (huffmanGroups<2 || huffmanGroups>6) throw DecompressionError(); + + uint32_t selectorsUsed=readBits(15); + if (!selectorsUsed) throw DecompressionError(); + + MemoryBuffer huffmanSelectorList(selectorsUsed); + + auto unMTF=[](uint8_t value,uint8_t map[])->uint8_t + { + uint8_t ret=map[value]; + if (value) + { + uint8_t tmp=map[value]; + for (uint32_t i=value;i;i--) + map[i]=map[i-1]; + map[0]=tmp; + } + return ret; + }; + + // create Huffman selectors + uint8_t selectorMTFMap[6]={0,1,2,3,4,5}; + + for (uint32_t i=0;i<selectorsUsed;i++) + { + uint8_t item=unMTF(selectorDecoder.decode(readBit),selectorMTFMap); + if (item>=huffmanGroups) throw DecompressionError(); + huffmanSelectorList[i]=item; + } + + typedef HuffmanDecoder<uint32_t> BZIP2Decoder; + std::vector<BZIP2Decoder> dataDecoders(huffmanGroups); + + // Create all tables + for (uint32_t i=0;i<huffmanGroups;i++) + { + uint8_t bitLengths[258]; + + uint32_t currentBits=readBits(5); + for (uint32_t j=0;j<numHuffmanItems;j++) + { + int32_t delta; + do + { + delta=deltaDecoder.decode(readBit); + currentBits+=delta; + } while (delta); + if (currentBits<1 || currentBits>20) throw DecompressionError(); + bitLengths[j]=currentBits; + } + + dataDecoders[i].createOrderlyHuffmanTable(bitLengths,numHuffmanItems); + } + + // Huffman decode + unRLE + unMTF + BZIP2Decoder *currentHuffmanDecoder=nullptr; + uint32_t currentHuffmanIndex=0; + uint8_t dataMTFMap[256]; + for (uint32_t i=0;i<numHuffmanItems-2;i++) dataMTFMap[i]=i; + + uint32_t currentRunLength=0; + uint32_t currentRLEWeight=1; + + auto decodeRLE=[&]() + { + if (currentRunLength) + { + if (currentBlockSize+currentRunLength>_blockSize) throw DecompressionError(); + for (uint32_t i=0;i<currentRunLength;i++) tmpBufferPtr[currentBlockSize++]=huffmanValues[dataMTFMap[0]]; + } + currentRunLength=0; + currentRLEWeight=1; + }; + + for (uint32_t streamIndex=0;;streamIndex++) + { + if (!(streamIndex%50)) + { + if (currentHuffmanIndex>=selectorsUsed) throw DecompressionError(); + currentHuffmanDecoder=&dataDecoders[huffmanSelectorList[currentHuffmanIndex++]]; + } + uint32_t symbolMTF=currentHuffmanDecoder->decode(readBit); + // stop marker is referenced only once, and it is the last one + // This means we do no have to un-MTF it for detection + if (symbolMTF==numHuffmanItems-1) break; + if (currentBlockSize>=_blockSize) throw DecompressionError(); + if (symbolMTF<2) + { + currentRunLength+=currentRLEWeight<<symbolMTF; + currentRLEWeight<<=1; + } else { + decodeRLE(); + uint8_t symbol=unMTF(symbolMTF-1,dataMTFMap); + if (currentBlockSize>=_blockSize) throw DecompressionError(); + tmpBufferPtr[currentBlockSize++]=huffmanValues[symbol]; + } + } + decodeRLE(); + if (currentPtr>=currentBlockSize) throw DecompressionError(); + } + + // inverse BWT + final RLE decoding. + // there are a few dark corners here as well + // 1. Can the stream end at 4 literals without count? I assume it is a valid optimization (and that this does not spillover to next block) + // 2. Can the RLE-step include counts 252 to 255 even if reference does not do them? I assume yes here as here as well + // 3. Can the stream be empty? We do not take issue here about that (that should be culled out earlier already) + uint32_t sums[256]; + for (uint32_t i=0;i<256;i++) sums[i]=0; + + for (uint32_t i=0;i<currentBlockSize;i++) + { + sums[tmpBufferPtr[i]]++; + } + + uint32_t rank[256]; + for (uint32_t tot=0,i=0;i<256;i++) + { + rank[i]=tot; + tot+=sums[i]; + } + + // not at all happy about the memory consumption, but it simplifies the implementation a lot + // and by sacrificing 4*size (size as in actual block size) we do not have to have slow search nor another temporary buffer + // since by calculating forward table we can do forward decoding of the data on the same pass as iBWT + // + // also, because I'm lazy + MemoryBuffer forwardIndex(currentBlockSize*sizeof(uint32_t)); + auto forwardIndexPtr=forwardIndex.cast<uint32_t>(); + for (uint32_t i=0;i<currentBlockSize;i++) + forwardIndexPtr[rank[tmpBufferPtr[i]]++]=i; + + // output + final RLE decoding + uint8_t currentCh=0; + uint32_t currentChCount=0; + auto outputByte=[&](uint8_t ch) + { + if (randomized && randomBit()) ch^=1; + if (!currentChCount) + { + currentCh=ch; + currentChCount=1; + } else { + if (ch==currentCh && currentChCount!=4) + { + currentChCount++; + } else { + auto outputBlock=[&](uint32_t count) + { + for (uint32_t i=0;i<count;i++) outputStream.writeByte(currentCh); + }; + + if (currentChCount==4) + { + outputBlock(uint32_t(ch)+4); + currentChCount=0; + } else { + outputBlock(currentChCount); + currentCh=ch; + currentChCount=1; + } + } + } + }; + + size_t destOffsetStart=outputStream.getOffset(); + + // and now the final iBWT + unRLE is easy... + for (uint32_t i=0;i<currentBlockSize;i++) + { + currentPtr=forwardIndexPtr[currentPtr]; + outputByte(tmpBufferPtr[currentPtr]); + } + // cleanup the state, a bit hackish way to do it + if (currentChCount) outputByte(currentChCount==4?0:~currentCh); + + if (verify) + calculateBlockCRC(destOffsetStart,outputStream.getOffset()-destOffsetStart); + + } else if (blockHdrHigh==0x17724538U && blockHdrLow==0x5090U) { + // end of blocks + uint32_t rawCRC=readBits(32); + if (verify && crc!=rawCRC) throw VerificationError(); + break; + } else throw DecompressionError(); + } + + if (!_rawSize) _rawSize=outputStream.getOffset(); + if (!_packedSize) _packedSize=inputStream.getOffset(); + if (_rawSize!=outputStream.getOffset()) throw DecompressionError(); +} + +void BZIP2Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + return decompressImpl(rawData,verify); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/BZIP2Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/BZIP2Decompressor.hpp new file mode 100644 index 00000000..f84cf06c --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/BZIP2Decompressor.hpp @@ -0,0 +1,44 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef BZIP2DECOMPRESSOR_HPP +#define BZIP2DECOMPRESSOR_HPP + +#include "Decompressor.hpp" +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class BZIP2Decompressor : public Decompressor, public XPKDecompressor +{ +public: + BZIP2Decompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); + BZIP2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + virtual ~BZIP2Decompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeader(uint32_t hdr) noexcept; + static bool detectHeaderXPK(uint32_t hdr) noexcept; + + static std::shared_ptr<Decompressor> create(const Buffer &packedData,bool exactSizeKnown,bool verify); + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + size_t _blockSize=0; + size_t _packedSize=0; + size_t _rawSize=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/BZIP2Table.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/BZIP2Table.hpp new file mode 100644 index 00000000..59a65092 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/BZIP2Table.hpp @@ -0,0 +1,78 @@ +/* +-------------------------------------------------------------------------- + +This program, "bzip2", the associated library "libbzip2", and all +documentation, are copyright (C) 1996-2010 Julian R Seward. All +rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Julian Seward, jseward@bzip.org +bzip2/libbzip2 version 1.0.6 of 6 September 2010 + +-------------------------------------------------------------------------- +*/ + +// content formatted, numbers -1 (i.e. this is altered version by clause 3) + static const uint16_t randomTable[512]={ + 619-1,720-1,127-1,481-1,931-1,816-1,813-1,233-1,566-1,247-1,985-1,724-1,205-1,454-1,863-1,491-1, + 741-1,242-1,949-1,214-1,733-1,859-1,335-1,708-1,621-1,574-1, 73-1,654-1,730-1,472-1,419-1,436-1, + 278-1,496-1,867-1,210-1,399-1,680-1,480-1, 51-1,878-1,465-1,811-1,169-1,869-1,675-1,611-1,697-1, + 867-1,561-1,862-1,687-1,507-1,283-1,482-1,129-1,807-1,591-1,733-1,623-1,150-1,238-1, 59-1,379-1, + 684-1,877-1,625-1,169-1,643-1,105-1,170-1,607-1,520-1,932-1,727-1,476-1,693-1,425-1,174-1,647-1, + 73-1,122-1,335-1,530-1,442-1,853-1,695-1,249-1,445-1,515-1,909-1,545-1,703-1,919-1,874-1,474-1, + 882-1,500-1,594-1,612-1,641-1,801-1,220-1,162-1,819-1,984-1,589-1,513-1,495-1,799-1,161-1,604-1, + 958-1,533-1,221-1,400-1,386-1,867-1,600-1,782-1,382-1,596-1,414-1,171-1,516-1,375-1,682-1,485-1, + 911-1,276-1, 98-1,553-1,163-1,354-1,666-1,933-1,424-1,341-1,533-1,870-1,227-1,730-1,475-1,186-1, + 263-1,647-1,537-1,686-1,600-1,224-1,469-1, 68-1,770-1,919-1,190-1,373-1,294-1,822-1,808-1,206-1, + 184-1,943-1,795-1,384-1,383-1,461-1,404-1,758-1,839-1,887-1,715-1, 67-1,618-1,276-1,204-1,918-1, + 873-1,777-1,604-1,560-1,951-1,160-1,578-1,722-1, 79-1,804-1, 96-1,409-1,713-1,940-1,652-1,934-1, + 970-1,447-1,318-1,353-1,859-1,672-1,112-1,785-1,645-1,863-1,803-1,350-1,139-1, 93-1,354-1, 99-1, + 820-1,908-1,609-1,772-1,154-1,274-1,580-1,184-1, 79-1,626-1,630-1,742-1,653-1,282-1,762-1,623-1, + 680-1, 81-1,927-1,626-1,789-1,125-1,411-1,521-1,938-1,300-1,821-1, 78-1,343-1,175-1,128-1,250-1, + 170-1,774-1,972-1,275-1,999-1,639-1,495-1, 78-1,352-1,126-1,857-1,956-1,358-1,619-1,580-1,124-1, + 737-1,594-1,701-1,612-1,669-1,112-1,134-1,694-1,363-1,992-1,809-1,743-1,168-1,974-1,944-1,375-1, + 748-1, 52-1,600-1,747-1,642-1,182-1,862-1, 81-1,344-1,805-1,988-1,739-1,511-1,655-1,814-1,334-1, + 249-1,515-1,897-1,955-1,664-1,981-1,649-1,113-1,974-1,459-1,893-1,228-1,433-1,837-1,553-1,268-1, + 926-1,240-1,102-1,654-1,459-1, 51-1,686-1,754-1,806-1,760-1,493-1,403-1,415-1,394-1,687-1,700-1, + 946-1,670-1,656-1,610-1,738-1,392-1,760-1,799-1,887-1,653-1,978-1,321-1,576-1,617-1,626-1,502-1, + 894-1,679-1,243-1,440-1,680-1,879-1,194-1,572-1,640-1,724-1,926-1, 56-1,204-1,700-1,707-1,151-1, + 457-1,449-1,797-1,195-1,791-1,558-1,945-1,679-1,297-1, 59-1, 87-1,824-1,713-1,663-1,412-1,693-1, + 342-1,606-1,134-1,108-1,571-1,364-1,631-1,212-1,174-1,643-1,304-1,329-1,343-1, 97-1,430-1,751-1, + 497-1,314-1,983-1,374-1,822-1,928-1,140-1,206-1, 73-1,263-1,980-1,736-1,876-1,478-1,430-1,305-1, + 170-1,514-1,364-1,692-1,829-1, 82-1,855-1,953-1,676-1,246-1,369-1,970-1,294-1,750-1,807-1,827-1, + 150-1,790-1,288-1,923-1,804-1,378-1,215-1,828-1,592-1,281-1,565-1,555-1,710-1, 82-1,896-1,831-1, + 547-1,261-1,524-1,462-1,293-1,465-1,502-1, 56-1,661-1,821-1,976-1,991-1,658-1,869-1,905-1,758-1, + 745-1,193-1,768-1,550-1,608-1,933-1,378-1,286-1,215-1,979-1,792-1,961-1, 61-1,688-1,793-1,644-1, + 986-1,403-1,106-1,366-1,905-1,644-1,372-1,567-1,466-1,434-1,645-1,210-1,389-1,550-1,919-1,135-1, + 780-1,773-1,635-1,389-1,707-1,100-1,626-1,958-1,165-1,504-1,920-1,176-1,193-1,713-1,857-1,265-1, + 203-1, 50-1,668-1,108-1,645-1,990-1,626-1,197-1,510-1,357-1,358-1,850-1,858-1,364-1,936-1,638-1}; diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/CBR0Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/CBR0Decompressor.cpp new file mode 100644 index 00000000..3d4a65c7 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/CBR0Decompressor.cpp @@ -0,0 +1,64 @@ +/* Copyright (C) Teemu Suutari */ + +#include "CBR0Decompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool CBR0Decompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + // CBR1 is practical joke: it is the same as CBR0 but with ID changed + return hdr==FourCC("CBR0") || hdr==FourCC("CBR1"); +} + +std::shared_ptr<XPKDecompressor> CBR0Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<CBR0Decompressor>(hdr,recursionLevel,packedData,state,verify); +} + +CBR0Decompressor::CBR0Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData), + _isCBR0(hdr==FourCC("CBR0")) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +CBR0Decompressor::~CBR0Decompressor() +{ + // nothing needed +} + +const std::string &CBR0Decompressor::getSubName() const noexcept +{ + static std::string nameCBR0="XPK-CBR0: RLE-compressor"; + static std::string nameCBR1="XPK-CBR1: RLE-compressor"; + return (_isCBR0)?nameCBR0:nameCBR1; +} + +void CBR0Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + // barely different than RLEN, however the count is well defined here. + while (!outputStream.eof()) + { + uint32_t count=inputStream.readByte(); + if (count<128) + { + count++; + for (uint32_t i=0;i<count;i++) outputStream.writeByte(inputStream.readByte()); + } else { + count=257-count; + uint8_t ch=inputStream.readByte(); + for (uint32_t i=0;i<count;i++) outputStream.writeByte(ch); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/CBR0Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/CBR0Decompressor.hpp new file mode 100644 index 00000000..34e71ec5 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/CBR0Decompressor.hpp @@ -0,0 +1,32 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef CBR0DECOMPRESSOR_HPP +#define CBR0DECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class CBR0Decompressor : public XPKDecompressor +{ +public: + CBR0Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~CBR0Decompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + bool _isCBR0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/CRMDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/CRMDecompressor.cpp new file mode 100644 index 00000000..a866d452 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/CRMDecompressor.cpp @@ -0,0 +1,245 @@ +/* Copyright (C) Teemu Suutari */ + +#include "CRMDecompressor.hpp" +#include "HuffmanDecoder.hpp" +#include "DLTADecode.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/OverflowCheck.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool CRMDecompressor::detectHeader(uint32_t hdr) noexcept +{ + switch (hdr) + { + case FourCC("CrM!"): + case FourCC("CrM2"): + case FourCC("Crm!"): + case FourCC("Crm2"): + return true; + + default: + return false; + } +} + +bool CRMDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("CRM2") || hdr==FourCC("CRMS"); +} + +std::shared_ptr<Decompressor> CRMDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) +{ + return std::make_shared<CRMDecompressor>(packedData,0,verify); +} + +std::shared_ptr<XPKDecompressor> CRMDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<CRMDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +CRMDecompressor::CRMDecompressor(const Buffer &packedData,uint32_t recursionLevel,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + uint32_t hdr=packedData.readBE32(0); + if (!detectHeader(hdr) || packedData.size()<20) + throw Decompressor::InvalidFormatError(); + + _rawSize=packedData.readBE32(6); + _packedSize=packedData.readBE32(10); + if (!_rawSize || !_packedSize || + _rawSize>getMaxRawSize() || _packedSize>getMaxPackedSize() || + OverflowCheck::sum(_packedSize,14U)>packedData.size()) throw Decompressor::InvalidFormatError(); + if (((hdr>>8)&0xff)=='m') _isSampled=true; + if ((hdr&0xff)=='2') _isLZH=true; +} + +CRMDecompressor::CRMDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + CRMDecompressor(packedData,recursionLevel,verify) +{ + _isXPKDelta=(hdr==FourCC("CRMS")); +} + +CRMDecompressor::~CRMDecompressor() +{ + // nothing needed +} + +const std::string &CRMDecompressor::getName() const noexcept +{ + static std::string names[4]={ + "CrM!: Crunch-Mania standard-mode", + "Crm!: Crunch-Mania standard-mode, sampled", + "CrM2: Crunch-Mania LZH-mode", + "Crm2: Crunch-Mania LZH-mode, sampled"}; + return names[(_isLZH?2:0)+(_isSampled?1:0)]; +} + +const std::string &CRMDecompressor::getSubName() const noexcept +{ + // the XPK-id is not used in decompressing process, + // but there is a real id inside the stream + // This means we can have frankenstein configurations, + // although in practice we don't + static std::string names[2]={ + "XPK-CRM2: Crunch-Mania LZH-mode", + "XPK-CRMS: Crunch-Mania LZH-mode, sampled"}; + return names[(_isXPKDelta?1:0)]; +} + +size_t CRMDecompressor::getPackedSize() const noexcept +{ + return _packedSize+14; +} + +size_t CRMDecompressor::getRawSize() const noexcept +{ + return _rawSize; +} + +void CRMDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + if (rawData.size()<_rawSize) throw Decompressor::DecompressionError(); + + BackwardInputStream inputStream(_packedData,14,_packedSize+14-6); + LSBBitReader<BackwardInputStream> bitReader(inputStream); + { + // There are empty bits?!? at the start of the stream. take them out + size_t bufOffset=_packedSize+14-6; + uint32_t originalBitsContent=_packedData.readBE32(bufOffset); + uint16_t originalShift=_packedData.readBE16(bufOffset+4); + uint8_t bufBitsLength=originalShift+16; + uint32_t bufBitsContent=originalBitsContent>>(16-originalShift); + bitReader.reset(bufBitsContent,bufBitsLength); + } + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + BackwardOutputStream outputStream(rawData,0,_rawSize); + + if (_isLZH) + { + typedef HuffmanDecoder<uint32_t> CRMHuffmanDecoder; + + auto readHuffmanTable=[&](CRMHuffmanDecoder &dec,uint32_t codeLength) + { + uint32_t maxDepth=readBits(4); + if (!maxDepth) throw Decompressor::DecompressionError(); + uint32_t lengthTable[15]; + for (uint32_t i=0;i<maxDepth;i++) + lengthTable[i]=readBits(std::min(i+1,codeLength)); + uint32_t code=0; + for (uint32_t depth=1;depth<=maxDepth;depth++) + { + for (uint32_t i=0;i<lengthTable[depth-1];i++) + { + uint32_t value=readBits(codeLength); + dec.insert(HuffmanCode<uint32_t>{depth,code>>(maxDepth-depth),value}); + code+=1<<(maxDepth-depth); + } + } + }; + + + do { + CRMHuffmanDecoder lengthDecoder,distanceDecoder; + readHuffmanTable(lengthDecoder,9); + readHuffmanTable(distanceDecoder,4); + + uint32_t items=readBits(16)+1; + for (uint32_t i=0;i<items;i++) + { + uint32_t count=lengthDecoder.decode(readBit); + if (count&0x100) + { + outputStream.writeByte(count); + } else { + count+=3; + + uint32_t distanceBits=distanceDecoder.decode(readBit); + uint32_t distance; + if (!distanceBits) + { + distance=readBits(1)+1; + } else { + distance=(readBits(distanceBits)|(1<<distanceBits))+1; + } + outputStream.copy(distance,count); + } + } + } while (readBit()); + } else { + HuffmanDecoder<uint8_t> lengthDecoder + { + HuffmanCode<uint8_t>{1,0b000,0}, + HuffmanCode<uint8_t>{2,0b010,1}, + HuffmanCode<uint8_t>{3,0b110,2}, + HuffmanCode<uint8_t>{3,0b111,3} + }; + + HuffmanDecoder<uint8_t> distanceDecoder + { + HuffmanCode<uint8_t>{1,0b00,0}, + HuffmanCode<uint8_t>{2,0b10,1}, + HuffmanCode<uint8_t>{2,0b11,2} + }; + + while (!outputStream.eof()) + { + if (readBit()) + { + outputStream.writeByte(readBits(8)); + } else { + uint8_t lengthIndex=lengthDecoder.decode(readBit); + + static const uint8_t lengthBits[4]={1,2,4,8}; + static const uint32_t lengthAdditions[4]={2,4,8,24}; + uint32_t count=readBits(lengthBits[lengthIndex])+lengthAdditions[lengthIndex]; + if (count==23) + { + if (readBit()) + { + count=readBits(5)+15; + } else { + count=readBits(14)+15; + } + for (uint32_t i=0;i<count;i++) + outputStream.writeByte(readBits(8)); + } else { + if (count>23) count--; + + uint8_t distanceIndex=distanceDecoder.decode(readBit); + + static const uint8_t distanceBits[3]={9,5,14}; + static const uint32_t distanceAdditions[3]={32,0,544}; + uint32_t distance=readBits(distanceBits[distanceIndex])+distanceAdditions[distanceIndex]; + + outputStream.copy(distance,count); + } + } + } + } + + if (!outputStream.eof()) throw Decompressor::DecompressionError(); + if (_isSampled) + DLTADecode::decode(rawData,rawData,0,_rawSize); +} + +void CRMDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); + return decompressImpl(rawData,verify); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/CRMDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/CRMDecompressor.hpp new file mode 100644 index 00000000..1b4c4f35 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/CRMDecompressor.hpp @@ -0,0 +1,46 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef CRMDECOMPRESSOR_HPP +#define CRMDECOMPRESSOR_HPP + +#include "Decompressor.hpp" +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class CRMDecompressor : public Decompressor, public XPKDecompressor +{ +public: + CRMDecompressor(const Buffer &packedData,uint32_t recursionLevel,bool verify); + CRMDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + virtual ~CRMDecompressor(); + + virtual const std::string &getName() const noexcept override final; + virtual const std::string &getSubName() const noexcept override final; + + virtual size_t getPackedSize() const noexcept override final; + virtual size_t getRawSize() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeader(uint32_t hdr) noexcept; + static bool detectHeaderXPK(uint32_t hdr) noexcept; + + static std::shared_ptr<Decompressor> create(const Buffer &packedData,bool exactSizeKnown,bool verify); + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _packedSize=0; + uint32_t _rawSize=0; + bool _isLZH=false; // "normal" compression or LZH compression + bool _isSampled=false; // normal or "sampled" i.e. obsfuscated + bool _isXPKDelta=false; // If delta encoding defined in XPK +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/CYB2Decoder.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/CYB2Decoder.cpp new file mode 100644 index 00000000..2deca68f --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/CYB2Decoder.cpp @@ -0,0 +1,62 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstring> + +#include "common/SubBuffer.hpp" +#include "CYB2Decoder.hpp" +#include "XPKMain.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool CYB2Decoder::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("CYB2"); +} + +std::shared_ptr<XPKDecompressor> CYB2Decoder::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<CYB2Decoder>(hdr,recursionLevel,packedData,state,verify); +} + +CYB2Decoder::CYB2Decoder(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr) || _packedData.size()<=10) throw Decompressor::InvalidFormatError(); + _blockHeader=_packedData.readBE32(0); + // after the block header, the next 6 bytes seem to be + // 00 64 00 00 00 00 + // Those bytes do not seem to be terribly important though... + + if (verify) + { + // trigger child checks... + ConstSubBuffer blockData(_packedData,10,_packedData.size()-10); + std::shared_ptr<XPKDecompressor::State> state; + auto sub=XPKMain::createDecompressor(_blockHeader,_recursionLevel+1,blockData,state,true); + } +} + +CYB2Decoder::~CYB2Decoder() +{ + // nothing needed +} + +const std::string &CYB2Decoder::getSubName() const noexcept +{ + static std::string name="XPK-CYB2: xpkCybPrefs container"; + return name; +} + +void CYB2Decoder::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ConstSubBuffer blockData(_packedData,10,_packedData.size()-10); + std::shared_ptr<XPKDecompressor::State> state; + auto sub=XPKMain::createDecompressor(_blockHeader,_recursionLevel+1,blockData,state,verify); + sub->decompressImpl(rawData,previousData,verify); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/CYB2Decoder.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/CYB2Decoder.hpp new file mode 100644 index 00000000..33f7f56f --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/CYB2Decoder.hpp @@ -0,0 +1,33 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef CYB2DECODER_HPP +#define CYB2DECODER_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class CYB2Decoder : public XPKDecompressor +{ +public: + CYB2Decoder(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~CYB2Decoder(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _blockHeader; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/DEFLATEDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DEFLATEDecompressor.cpp new file mode 100644 index 00000000..d4dc635c --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DEFLATEDecompressor.cpp @@ -0,0 +1,386 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstdint> +#include <cstring> + +#include "DEFLATEDecompressor.hpp" +#include "HuffmanDecoder.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/CRC32.hpp" +#include "common/OverflowCheck.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +static uint32_t Adler32(const Buffer &buffer,size_t offset,size_t len) +{ + if (!len || OverflowCheck::sum(offset,len)>buffer.size()) throw Buffer::OutOfBoundsError(); + const uint8_t *ptr=buffer.data()+offset; + + uint32_t s1=1,s2=0; + for (size_t i=0;i<len;i++) + { + s1+=ptr[i]; + if (s1>=65521) s1-=65521; + s2+=s1; + if (s2>=65521) s2-=65521; + } + return (s2<<16)|s1; +} + +bool DEFLATEDecompressor::detectHeader(uint32_t hdr) noexcept +{ + return ((hdr>>16)==0x1f8b); +} + +bool DEFLATEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return (hdr==FourCC("GZIP")); +} + +std::shared_ptr<Decompressor> DEFLATEDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) +{ + return std::make_shared<DEFLATEDecompressor>(packedData,exactSizeKnown,verify); +} + +std::shared_ptr<XPKDecompressor> DEFLATEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<DEFLATEDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +bool DEFLATEDecompressor::detectZLib() +{ + if (_packedData.size()<6) return false; + + // no knowledge about rawSize, before decompression + // packedSize told by decompressor + _packedSize=uint32_t(_packedData.size()); + _packedOffset=2; + + uint8_t cm=_packedData.read8(0); + if ((cm&0xf)!=8) return false; + if ((cm&0xf0)>0x70) return false; + + uint8_t flags=_packedData.read8(1); + if (flags&0x20) + { + if (_packedSize<8) return false; + _packedOffset+=4; + } + + if (((uint16_t(cm)<<8)|uint16_t(flags))%31) return false; + + _type=Type::ZLib; + return true; +} + +DEFLATEDecompressor::DEFLATEDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : + _packedData(packedData), + _exactSizeKnown(exactSizeKnown) +{ + if (_packedData.size()<18) throw InvalidFormatError(); + uint32_t hdr=_packedData.readBE32(0); + if (!detectHeader(hdr)) throw InvalidFormatError(); + + uint8_t cm=_packedData.read8(2); + if (cm!=8) throw InvalidFormatError();; + + uint8_t flags=_packedData.read8(3); + if (flags&0xe0) throw InvalidFormatError();; + + uint32_t currentOffset=10; + + if (flags&4) + { + uint16_t xlen=_packedData.readLE16(currentOffset); + currentOffset+=uint32_t(xlen)+2; + } + + auto skipString=[&]() + { + uint8_t ch; + do { + ch=_packedData.read8(currentOffset); + currentOffset++; + } while (ch); + }; + + if (flags&8) skipString(); // FNAME + if (flags&16) skipString(); // FCOMMENT + + if (flags&2) currentOffset+=2; // FHCRC, not using that since it is only for header + _packedOffset=currentOffset; + + if (OverflowCheck::sum(currentOffset,8U)>_packedData.size()) throw InvalidFormatError(); + + if (_exactSizeKnown) + { + _packedSize=_packedData.size(); + _rawSize=_packedData.readLE32(_packedData.size()-4); + if (!_rawSize) throw InvalidFormatError(); + } + + _type=Type::GZIP; +} + +DEFLATEDecompressor::DEFLATEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectZLib()) + { + _packedSize=packedData.size(); + _packedOffset=0; + _type=Type::Raw; + } +} + +DEFLATEDecompressor::DEFLATEDecompressor(const Buffer &packedData,size_t packedSize,size_t rawSize,bool isZlib,bool verify,bool deflate64) : + _packedData(packedData), + _deflate64(deflate64) +{ + _packedSize=packedSize; + if (_packedSize>_packedData.size()) throw InvalidFormatError(); + if (isZlib) + { + // if it is not real zlib-stream fail. + if (!detectZLib()) throw InvalidFormatError(); + } else { + // raw stream + _packedOffset=0; + _rawSize=rawSize; + _type=Type::Raw; + } +} + +DEFLATEDecompressor::~DEFLATEDecompressor() +{ + // nothing needed +} + + +const std::string &DEFLATEDecompressor::getName() const noexcept +{ + static std::string names[3]={ + "gzip: Deflate", + "zlib: Deflate", + "raw: Deflate/Deflate64"}; + return names[static_cast<uint32_t>(_type)]; +} + +const std::string &DEFLATEDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-GZIP: Deflate"; + return name; +} + +size_t DEFLATEDecompressor::getPackedSize() const noexcept +{ + // no way to know before decompressing + return _packedSize; +} + + +size_t DEFLATEDecompressor::getRawSize() const noexcept +{ + // same thing, decompression needed first + return _rawSize; +} + +void DEFLATEDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + size_t packedSize=_packedSize?_packedSize:_packedData.size(); + size_t rawSize=_rawSize?_rawSize:rawData.size(); + + ForwardInputStream inputStream(_packedData,_packedOffset,packedSize); + LSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawSize); + + bool final; + do { + final=readBit(); + uint8_t blockType=readBits(2); + if (!blockType) + { + bitReader.reset(); + uint16_t len=inputStream.readByte(); + len|=uint16_t(inputStream.readByte())<<8; + uint16_t nlen=inputStream.readByte(); + nlen|=uint16_t(inputStream.readByte())<<8; + if (len!=(nlen^0xffffU)) throw DecompressionError(); + outputStream.produce(inputStream.consume(len),len); + } else if (blockType==1 || blockType==2) { + typedef HuffmanDecoder<int32_t> DEFLATEDecoder; + DEFLATEDecoder llDecoder; + DEFLATEDecoder distanceDecoder; + + if (blockType==1) + { + for (uint32_t i=0;i<24;i++) llDecoder.insert(HuffmanCode<int32_t>{7,i,int32_t(i+256)}); + for (uint32_t i=0;i<144;i++) llDecoder.insert(HuffmanCode<int32_t>{8,i+0x30,int32_t(i)}); + for (uint32_t i=0;i<8;i++) llDecoder.insert(HuffmanCode<int32_t>{8,i+0xc0,int32_t(i+280)}); + for (uint32_t i=0;i<112;i++) llDecoder.insert(HuffmanCode<int32_t>{9,i+0x190,int32_t(i+144)}); + + for (uint32_t i=0;i<32;i++) distanceDecoder.insert(HuffmanCode<int32_t>{5,i,int32_t(i)}); + } else { + uint32_t hlit=readBits(5)+257; + // lets just error here, it is simpler + if (hlit>=287) throw DecompressionError(); + uint32_t hdist=readBits(5)+1; + uint32_t hclen=readBits(4)+4; + + uint8_t lengthTable[19]; + for (uint32_t i=0;i<19;i++) lengthTable[i]=0; + static const uint8_t lengthTableOrder[19]={ + 16,17,18, 0, 8, 7, 9, 6, + 10, 5,11, 4,12, 3,13, 2, + 14, 1,15}; + for (uint32_t i=0;i<hclen;i++) lengthTable[lengthTableOrder[i]]=readBits(3); + + DEFLATEDecoder bitLengthDecoder; + bitLengthDecoder.createOrderlyHuffmanTable(lengthTable,19); // 19 and not hclen due to reordering + + // can the previous code flow from ll to distance table? + // specification does not say and treats the two almost as combined. + // So let previous code flow + + uint8_t llTableBits[286]; + uint8_t distanceTableBits[32]; + + uint8_t prevValue=0; + uint32_t i=0; + while (i<hlit+hdist) + { + auto insert=[&](uint8_t value) + { + if (i>=hlit+hdist) throw DecompressionError(); + if (i>=hlit) distanceTableBits[i-hlit]=value; + else llTableBits[i]=value; + prevValue=value; + i++; + }; + + int32_t code=bitLengthDecoder.decode(readBit); + if (code<16) { + insert(code); + } else switch (code) { + case 16: + if (i) + { + uint32_t count=readBits(2)+3; + for (uint32_t j=0;j<count;j++) insert(prevValue); + } else throw DecompressionError(); + break; + + case 17: + for (uint32_t count=readBits(3)+3;count;count--) insert(0); + break; + + case 18: + for (uint32_t count=readBits(7)+11;count;count--) insert(0); + break; + + default: + throw DecompressionError(); + } + + } + + llDecoder.createOrderlyHuffmanTable(llTableBits,hlit); + distanceDecoder.createOrderlyHuffmanTable(distanceTableBits,hdist); + } + + // and now decode + for (;;) + { + int32_t code=llDecoder.decode(readBit); + if (code<256) { + outputStream.writeByte(code); + } else if (code==256) { + break; + } else { + static const uint32_t lengthAdditions[29]={ + 3,4,5,6,7,8,9,10, + 11,13,15,17, + 19,23,27,31, + 35,43,51,59, + 67,83,99,115, + 131,163,195,227, + 258}; + static const uint32_t lengthBits[29]={ + 0,0,0,0,0,0,0,0, + 1,1,1,1,2,2,2,2, + 3,3,3,3,4,4,4,4, + 5,5,5,5, + 0}; + uint32_t count=(_deflate64&&code==285)?readBits(16)+3:(readBits(lengthBits[code-257])+lengthAdditions[code-257]); + int32_t distCode=distanceDecoder.decode(readBit); + if (distCode<0 || distCode>(_deflate64?31:29)) throw DecompressionError(); + static const uint32_t distanceAdditions[32]={ + 1,2,3,4,5,7,9,13, + 0x11,0x19,0x21,0x31,0x41,0x61,0x81,0xc1, + 0x101,0x181,0x201,0x301,0x401,0x601,0x801,0xc01, + 0x1001,0x1801,0x2001,0x3001,0x4001,0x6001, + 0x8001,0xc001}; + static const uint32_t distanceBits[32]={ + 0,0,0,0,1,1,2,2, + 3,3,4,4,5,5,6,6, + 7,7,8,8,9,9,10,10, + 11,11,12,12,13,13, + 14,14}; + uint32_t distance=readBits(distanceBits[distCode])+distanceAdditions[distCode]; + outputStream.copy(distance,count); + } + } + } else { + throw DecompressionError(); + } + } while (!final); + + if (!_rawSize) _rawSize=outputStream.getOffset(); + if (_type==Type::GZIP) + { + if (OverflowCheck::sum(inputStream.getOffset(),8U)>packedSize) throw DecompressionError(); + if (!_packedSize) + _packedSize=inputStream.getOffset()+8; + } else if (_type==Type::ZLib) { + if (OverflowCheck::sum(inputStream.getOffset(),4U)>packedSize) throw DecompressionError(); + if (!_packedSize) + _packedSize=inputStream.getOffset()+4; + } else { + if (!_packedSize) + _packedSize=inputStream.getOffset(); + } + if (_rawSize!=outputStream.getOffset()) throw DecompressionError(); + + if (verify) + { + if (_type==Type::GZIP) + { + uint32_t crc=_packedData.readLE32(inputStream.getOffset()); + if (CRC32(rawData,0,_rawSize,0)!=crc) throw VerificationError(); + } else if (_type==Type::ZLib) { + uint32_t adler=_packedData.readBE32(inputStream.getOffset()); + if (Adler32(rawData,0,_rawSize)!=adler) throw VerificationError(); + } + } +} + +void DEFLATEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + decompressImpl(rawData,verify); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/DEFLATEDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DEFLATEDecompressor.hpp new file mode 100644 index 00000000..b4ee7a45 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DEFLATEDecompressor.hpp @@ -0,0 +1,57 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef DEFLATEDECOMPRESSOR_HPP +#define DEFLATEDECOMPRESSOR_HPP + +#include "Decompressor.hpp" +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class DEFLATEDecompressor : public Decompressor, public XPKDecompressor +{ +public: + DEFLATEDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); + DEFLATEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + DEFLATEDecompressor(const Buffer &packedData,size_t packedSize,size_t rawSize,bool isZlib,bool verify,bool deflate64); // zlib or completely raw stream + virtual ~DEFLATEDecompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeader(uint32_t hdr) noexcept; + static bool detectHeaderXPK(uint32_t hdr) noexcept; + + static std::shared_ptr<Decompressor> create(const Buffer &packedData,bool exactSizeKnown,bool verify); + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + bool detectZLib(); + + enum class Type + { + GZIP=0, + ZLib, + Raw + }; + + const Buffer &_packedData; + + size_t _packedSize=0; + size_t _packedOffset=0; + size_t _rawSize=0; + Type _type; + bool _exactSizeKnown=true; + bool _deflate64=false; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/DLTADecode.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DLTADecode.cpp new file mode 100644 index 00000000..5cd8f1ce --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DLTADecode.cpp @@ -0,0 +1,62 @@ +/* Copyright (C) Teemu Suutari */ + +#include "DLTADecode.hpp" + +#include "common/OverflowCheck.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool DLTADecode::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("DLTA"); +} + +std::shared_ptr<XPKDecompressor> DLTADecode::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<DLTADecode>(hdr,recursionLevel,packedData,state,verify); +} + +DLTADecode::DLTADecode(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +DLTADecode::~DLTADecode() +{ + // nothing needed +} + +const std::string &DLTADecode::getSubName() const noexcept +{ + static std::string name="XPK-DLTA: Delta encoding"; + return name; +} + +void DLTADecode::decode(Buffer &bufferDest,const Buffer &bufferSrc,size_t offset,size_t size) +{ + if (OverflowCheck::sum(offset,size)>bufferSrc.size()) throw Buffer::OutOfBoundsError(); + if (OverflowCheck::sum(offset,size)>bufferDest.size()) throw Buffer::OutOfBoundsError(); + const uint8_t *src=bufferSrc.data()+offset; + uint8_t *dest=bufferDest.data()+offset; + + uint8_t ctr=0; + for (size_t i=0;i<size;i++) + { + ctr+=src[i]; + dest[i]=ctr; + } +} + + +void DLTADecode::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + if (rawData.size()<_packedData.size()) throw Decompressor::DecompressionError(); + decode(rawData,_packedData,0,_packedData.size()); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/DLTADecode.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DLTADecode.hpp new file mode 100644 index 00000000..d6bf7076 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DLTADecode.hpp @@ -0,0 +1,35 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef DLTADECODE_HPP +#define DLTADECODE_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + + +class DLTADecode : public XPKDecompressor +{ +public: + DLTADecode(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~DLTADecode(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + // static method for easy external usage. Buffers can be the same for in-place replacement + static void decode(Buffer &bufferDest,const Buffer &bufferSrc,size_t offset,size_t size); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/DMSDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DMSDecompressor.cpp new file mode 100644 index 00000000..ded83c6c --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DMSDecompressor.cpp @@ -0,0 +1,718 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstdint> +#include <cstring> + +#include "DMSDecompressor.hpp" + +#include "HuffmanDecoder.hpp" +#include "DynamicHuffmanDecoder.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" + +#include "common/MemoryBuffer.hpp" +#include "common/CRC16.hpp" +#include "common/OverflowCheck.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool DMSDecompressor::detectHeader(uint32_t hdr) noexcept +{ + return hdr==FourCC("DMS!"); +} + +std::shared_ptr<Decompressor> DMSDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) +{ + return std::make_shared<DMSDecompressor>(packedData,verify); +} + +DMSDecompressor::DMSDecompressor(const Buffer &packedData,bool verify) : + _packedData(packedData) +{ + uint32_t hdr=packedData.readBE32(0); + if (!detectHeader(hdr) || packedData.size()<56) throw InvalidFormatError(); + + if (verify && CRC16(packedData,4,50,0)!=packedData.readBE16(54)) + throw VerificationError(); + + uint16_t info=packedData.readBE16(10); + _isObsfuscated=info&2; // using 16 bit key is not encryption, it is obsfuscation + _isHD=info&16; + if (info&32) throw InvalidFormatError(); // MS-DOS disk + + // packed data in header is useless, as is rawsize and track numbers + // they are not always filled + + if (packedData.readBE16(50)>6) throw InvalidFormatError(); // either FMS or unknown + + // now calculate the real packed size, including headers + uint32_t offset=56; + uint32_t accountedSize=0; + uint32_t lastTrackSize=0; + uint32_t numTracks=0; + uint32_t minTrack=80; + uint32_t prevTrack=0; + while (offset+20<packedData.size()) + { + if (_packedData.readBE16(offset)!=MultiChar2("TR")) + { + // secondary exit criteria, should not be like this, if the header would be trustworthy + if (!accountedSize) throw InvalidFormatError(); + break; + } + uint32_t trackNo=_packedData.readBE16(offset+2); + // lets not go backwards on tracks! + if (trackNo<prevTrack) break; + + // header check + if (verify && CRC16(packedData,offset,18,0)!=packedData.readBE16(offset+18)) + throw VerificationError(); + + uint8_t mode=_packedData.read8(offset+13); + if (mode>6) throw InvalidFormatError(); + static const uint32_t contextSizes[7]={0,0,256,16384,16384,4096,8192}; + _contextBufferSize=std::max(_contextBufferSize,contextSizes[mode]); + + uint8_t flags=_packedData.read8(offset+12); + if ((mode>=2 && mode<=4) || (mode>=5 && (flags&4))) + { + _tmpBufferSize=std::max(_tmpBufferSize,uint32_t(_packedData.readBE16(offset+8))); + } + uint32_t packedChunkLength=packedData.readBE16(offset+6); + if (OverflowCheck::sum(offset,20U,packedChunkLength)>packedData.size()) + throw InvalidFormatError(); + if (verify && CRC16(packedData,offset+20,packedChunkLength,0)!=packedData.readBE16(offset+16)) + throw VerificationError(); + + if (trackNo<80) + { + if (trackNo>=numTracks) lastTrackSize=_packedData.readBE16(offset+10); + minTrack=std::min(minTrack,trackNo); + numTracks=std::max(numTracks,trackNo); + prevTrack=trackNo; + } + + offset+=packedChunkLength+20; + accountedSize+=packedChunkLength; + // this is the real exit critea, unfortunately + if (trackNo>=79 && trackNo<0x8000U) break; + } + uint32_t trackSize=(_isHD)?22528:11264; + _rawOffset=minTrack*trackSize; + if (minTrack>=numTracks) + throw InvalidFormatError(); + _minTrack=minTrack; + _rawSize=(numTracks-minTrack)*trackSize+lastTrackSize; + _imageSize=trackSize*80; + + _packedSize=offset; + if (_packedSize>getMaxPackedSize()) + throw InvalidFormatError(); +} + +DMSDecompressor::~DMSDecompressor() +{ + // nothing needed +} + +const std::string &DMSDecompressor::getName() const noexcept +{ + static std::string name="DMS: Disk Masher System"; + return name; +} + +size_t DMSDecompressor::getPackedSize() const noexcept +{ + return _packedSize; +} + +size_t DMSDecompressor::getRawSize() const noexcept +{ + return _rawSize; +} + +size_t DMSDecompressor::getImageSize() const noexcept +{ + return _imageSize; +} + +size_t DMSDecompressor::getImageOffset() const noexcept +{ + return _rawOffset; +} + +void DMSDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + uint32_t restartPosition=0; + if (!_isObsfuscated) + { + decompressImpl(rawData,verify,restartPosition); + } else { + while (restartPosition<0x20000U) + { + // more than single run here is really rare. It means that first track CRC succeeds + // but later something else fails + try + { + decompressImpl(rawData,verify,restartPosition); + return; + } catch (const Buffer::Error &) { + // just continue + } catch (const Decompressor::Error &) { + // just continue + } + restartPosition++; + } + throw DecompressionError(); + } +} + +// TODO: Too much state for a single method. too convoluted +// needs to be split +void DMSDecompressor::decompressImpl(Buffer &rawData,bool verify,uint32_t &restartPosition) +{ + if (rawData.size()<_rawSize) throw DecompressionError(); + MemoryBuffer contextBuffer(_contextBufferSize); + MemoryBuffer tmpBuffer(_tmpBufferSize); + uint32_t limitedDecompress=~0U; + + class UnObsfuscator + { + public: + UnObsfuscator(ForwardInputStream &inputStream) : + _inputStream(inputStream) + { + // nothing needed + } + + uint8_t readByte() + { + if (_inputStream.eof()) throw ShortInputError(); + uint8_t ch=_inputStream.readByte(); + if (!_obsfuscate) + { + return ch; + } else { + uint8_t ret=ch^_passAccumulator; + _passAccumulator=(_passAccumulator>>1)+uint16_t(ch); + return ret; + } + } + + void setCode(uint16_t passAccumulator) + { + _passAccumulator=passAccumulator; + } + + void setObsfuscate(bool obsfuscate) { _obsfuscate=obsfuscate; } + bool eof() const { return _inputStream.getOffset()==_inputStream.getEndOffset(); } + + // not cool, but works (does not need wraparound check) + bool eofMinus1() const { return _inputStream.getOffset()+1==_inputStream.getEndOffset(); } + bool eofMinus1Plus() const { return _inputStream.getOffset()+1>=_inputStream.getEndOffset(); } + bool eofMinus2Plus() const { return _inputStream.getOffset()+2>=_inputStream.getEndOffset(); } + + private: + ForwardInputStream &_inputStream; + bool _obsfuscate=false; + uint16_t _passAccumulator=0; + }; + + ForwardInputStream inputStream(_packedData,0,0); + UnObsfuscator inputUnObsfuscator(inputStream); + MSBBitReader<UnObsfuscator> bitReader(inputUnObsfuscator); + auto initInputStream=[&](const Buffer &buffer,uint32_t start,uint32_t length,bool obsfuscate) + { + inputStream=ForwardInputStream(buffer,start,OverflowCheck::sum(start,length)); + inputUnObsfuscator.setObsfuscate(obsfuscate); + bitReader.reset(); + }; + auto finishStream=[&]() + { + if (_isObsfuscated && limitedDecompress==~0U) + while (!inputUnObsfuscator.eof()) + inputUnObsfuscator.readByte(); + }; + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,0); + auto initOutputStream=[&](Buffer &buffer,uint32_t start,uint32_t length) + { + outputStream=ForwardOutputStream(buffer,start,OverflowCheck::sum(start,length)); + }; + + // fill unused tracks with zeros + std::memset(rawData.data(),0,_rawSize); + + auto checksum=[](const uint8_t *src,uint32_t srcLength)->uint16_t + { + uint16_t ret=0; + for (uint32_t i=0;i<srcLength;i++) ret+=uint16_t(src[i]); + return ret; + }; + + auto unpackNone=[&]() + { + for (uint32_t i=0;i<limitedDecompress&&!inputUnObsfuscator.eof();i++) + outputStream.writeByte(inputUnObsfuscator.readByte()); + }; + + // same as simple + auto unRLE=[&](bool lastCharMissing)->uint32_t + { + // hacky, hacky, hacky + while (!inputUnObsfuscator.eof()) + { + if (outputStream.getOffset()>=limitedDecompress) return 0; + if (lastCharMissing && inputUnObsfuscator.eofMinus1()) + { + if (outputStream.getOffset()+1!=outputStream.getEndOffset()) + throw DecompressionError(); + return 1; + } + uint8_t ch=inputUnObsfuscator.readByte(); + uint32_t count=1; + if (ch==0x90U) + { + if (inputUnObsfuscator.eof()) throw DecompressionError(); + if (lastCharMissing && inputUnObsfuscator.eofMinus1()) + { + // only possible way this can work + if (outputStream.getOffset()+1!=outputStream.getEndOffset()) throw DecompressionError(); + count=0; + } else count=inputUnObsfuscator.readByte(); + if (!count) + { + count=1; + } else { + + if (inputUnObsfuscator.eof()) throw DecompressionError(); + if (lastCharMissing && inputUnObsfuscator.eofMinus1()) + { + if (count==0xffU || outputStream.getOffset()+count!=outputStream.getEndOffset()) throw DecompressionError(); + return count; + } + ch=inputUnObsfuscator.readByte(); + } + if (count==0xffU) + { + if (inputUnObsfuscator.eofMinus1Plus()) throw DecompressionError(); + if (lastCharMissing && inputUnObsfuscator.eofMinus2Plus()) + { + count=uint32_t(outputStream.getEndOffset()-outputStream.getOffset()); + } else { + count=uint32_t(inputUnObsfuscator.readByte())<<8; + count|=inputUnObsfuscator.readByte(); + } + } + } + for (uint32_t i=0;i<count;i++) outputStream.writeByte(ch); + } + if (!outputStream.eof()) throw DecompressionError(); + return 0; + }; + + bool doInitContext=true; + uint8_t *contextBufferPtr=contextBuffer.data(); + // context used is 256 bytes + uint8_t quickContextLocation; + // context used is 16384 bytes + uint32_t mediumContextLocation; + uint32_t deepContextLocation; + // context used is 4096/8192 bytes + uint32_t heavyContextLocation; + std::unique_ptr<DynamicHuffmanDecoder<314>> deepDecoder; + auto initContext=[&]() + { + if (doInitContext) + { + if (_contextBufferSize) std::memset(contextBuffer.data(),0,_contextBufferSize); + quickContextLocation=251; + mediumContextLocation=16318; + deepContextLocation=16324; + deepDecoder.reset(); + heavyContextLocation=0; + doInitContext=false; + } + }; + + auto unpackQuick=[&]() + { + initContext(); + + while (!outputStream.eof()) + { + if (outputStream.getOffset()>=limitedDecompress) return; + if (readBits(1)) + { + outputStream.writeByte(contextBufferPtr[quickContextLocation++]=readBits(8)); + } else { + uint32_t count=readBits(2)+2; + uint8_t offset=quickContextLocation-readBits(8)-1; + for (uint32_t i=0;i<count;i++) + outputStream.writeByte(contextBufferPtr[quickContextLocation++]=contextBufferPtr[(i+offset)&0xffU]); + } + } + quickContextLocation+=5; + }; + + static const uint8_t lengthTable[256]={ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, + 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, + 10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11, + 12,12,12,12,13,13,13,13, 14,14,14,14,15,15,15,15, + 16,16,16,16,17,17,17,17, 18,18,18,18,19,19,19,19, + 20,20,20,20,21,21,21,21, 22,22,22,22,23,23,23,23, + 24,24,25,25,26,26,27,27, 28,28,29,29,30,30,31,31, + 32,32,33,33,34,34,35,35, 36,36,37,37,38,38,39,39, + 40,40,41,41,42,42,43,43, 44,44,45,45,46,46,47,47, + 48,49,50,51,52,53,54,55, 56,57,58,59,60,61,62,63}; + + static const uint8_t bitLengthTable[256]={ + 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, + 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, + 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, + 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, + 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, + 5,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5, + 5,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5, + 5,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5, + 5,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5, + 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, + 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, + 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, + 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7, + 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8}; + + auto decodeLengthValueHalf=[&](uint8_t code)->uint32_t + { + return (((code<<bitLengthTable[code])|readBits(bitLengthTable[code]))&0xffU); + }; + + auto decodeLengthValueFull=[&](uint8_t code)->uint32_t + { + return (uint32_t(lengthTable[code])<<8)| + uint32_t(((code<<bitLengthTable[code])|readBits(bitLengthTable[code]))&0xffU); + }; + + auto unpackMedium=[&]() + { + initContext(); + + while (!outputStream.eof()) + { + if (outputStream.getOffset()>=limitedDecompress) return; + if (readBits(1)) + { + outputStream.writeByte(contextBufferPtr[mediumContextLocation++]=readBits(8)); + mediumContextLocation&=0x3fffU; + } else { + uint32_t code=readBits(8); + uint32_t count=lengthTable[code]+3; + + uint32_t tmp=decodeLengthValueFull(decodeLengthValueHalf(code)); + + uint32_t offset=mediumContextLocation-tmp-1; + for (uint32_t i=0;i<count;i++) + { + outputStream.writeByte(contextBufferPtr[mediumContextLocation++]=contextBufferPtr[(i+offset)&0x3fffU]); + mediumContextLocation&=0x3fffU; + } + } + } + mediumContextLocation+=66; + mediumContextLocation&=0x3fffU; + }; + + auto unpackDeep=[&]() + { + initContext(); + if (!deepDecoder) deepDecoder=std::make_unique<DynamicHuffmanDecoder<314>>(); + + while (!outputStream.eof()) + { + if (outputStream.getOffset()>=limitedDecompress) return; + uint32_t symbol=deepDecoder->decode(readBit); + if (deepDecoder->getMaxFrequency()==0x8000U) deepDecoder->halve(); + deepDecoder->update(symbol); + if (symbol<256) + { + outputStream.writeByte(contextBufferPtr[deepContextLocation++]=symbol); + deepContextLocation&=0x3fffU; + } else { + uint32_t count=symbol-253; // minimum repeat is 3 + uint32_t offset=deepContextLocation-decodeLengthValueFull(readBits(8))-1; + + for (uint32_t i=0;i<count;i++) + { + outputStream.writeByte(contextBufferPtr[deepContextLocation++]=contextBufferPtr[(i+offset)&0x3fffU]); + deepContextLocation&=0x3fffU; + } + } + } + deepContextLocation+=60; + deepContextLocation&=0x3fffU; + }; + + // these are not part of the initContext like other methods + std::unique_ptr<OptionalHuffmanDecoder<uint32_t>> symbolDecoder,offsetDecoder; + bool heavyLastInitialized=false; // this is part of initContext on some implementations. screwy!!! + uint32_t heavyLastOffset; + auto unpackHeavy=[&](bool initTables,bool use8kDict) + { + initContext(); + // well, this works. Why this works? dunno + if (!heavyLastInitialized) + { + heavyLastOffset=use8kDict?0U:~0U; + heavyLastInitialized=true; + } + + auto readTable=[&](std::unique_ptr<OptionalHuffmanDecoder<uint32_t>> &decoder,uint32_t countBits,uint32_t valueBits) + { + decoder=std::make_unique<OptionalHuffmanDecoder<uint32_t>>(); + uint32_t count=readBits(countBits); + if (count) + { + uint8_t lengthBuffer[512]; + // in order to speed up the deObsfuscation, do not send the hopeless + // data into slow CreateOrderlyHuffmanTable + uint64_t sum=0; + for (uint32_t i=0;i<count;i++) + { + uint32_t bits=readBits(valueBits); + if (bits) + { + sum+=uint64_t(1U)<<(32-bits); + if (sum>(uint64_t(1U)<<32)) + throw DecompressionError(); + } + lengthBuffer[i]=bits; + } + decoder->createOrderlyHuffmanTable(lengthBuffer,count); + } else { + uint32_t index=readBits(countBits); + decoder->setEmpty(index); + } + }; + + if (initTables) + { + readTable(symbolDecoder,9,5); + readTable(offsetDecoder,5,4); + } + + uint32_t mask=use8kDict?0x1fffU:0xfffU; + uint32_t bitLength=use8kDict?14:13; + + while (!outputStream.eof()) + { + if (outputStream.getOffset()>=limitedDecompress) return; + uint32_t symbol=symbolDecoder->decode(readBit); + if (symbol<256) + { + outputStream.writeByte(contextBufferPtr[heavyContextLocation++]=symbol); + heavyContextLocation&=mask; + } else { + uint32_t count=symbol-253; // minimum repeat is 3 + uint32_t offsetLength=offsetDecoder->decode(readBit); + uint32_t rawOffset=heavyLastOffset; + if (offsetLength!=bitLength) + { + if (offsetLength) rawOffset=(1<<(offsetLength-1))|readBits(offsetLength-1); + else rawOffset=0; + heavyLastOffset=rawOffset; + } + uint32_t offset=heavyContextLocation-rawOffset-1; + for (uint32_t i=0;i<count;i++) + { + outputStream.writeByte(contextBufferPtr[heavyContextLocation++]=contextBufferPtr[(i+offset)&mask]); + heavyContextLocation&=mask; + } + } + } + }; + + + uint32_t trackLength=(_isHD)?22528:11264; + for (uint32_t packedOffset=56,packedChunkLength=0;packedOffset!=_packedSize;packedOffset=OverflowCheck::sum(packedOffset,20U,packedChunkLength)) + { + // There are some info tracks, at -1 or 80. ignore those (if still present) + uint16_t trackNo=_packedData.readBE16(packedOffset+2); + packedChunkLength=_packedData.readBE16(packedOffset+6); + if (trackNo==80) break; // should not happen, this is already excluded + // even though only -1 should be used I've seen -2 as well. ignore all negatives + uint32_t tmpChunkLength=_packedData.readBE16(packedOffset+8); // after the first unpack (if twostage) + uint32_t rawChunkLength=_packedData.readBE16(packedOffset+10); // after final unpack + uint8_t flags=_packedData.read8(packedOffset+12); + uint8_t mode=_packedData.read8(packedOffset+13); + + // could affect context, but in practice they are separate, even though there is no explicit reset + // deal with decompression though... + if (trackNo>=0x8000U) + { + initInputStream(_packedData,packedOffset+20,packedChunkLength,_isObsfuscated); + finishStream(); + continue; + } + if (rawChunkLength>trackLength) throw DecompressionError(); + if (trackNo>80) throw DecompressionError(); // should not happen, already excluded + + uint32_t dataOffset=trackNo*trackLength; + if (_rawOffset>dataOffset) throw DecompressionError(); + + // this is screwy, but it is, what it is + auto processBlock=[&](bool doRLE,auto func,auto&&... params) + { + initInputStream(_packedData,packedOffset+20,packedChunkLength,_isObsfuscated); + if (doRLE) + { + try + { + initOutputStream(tmpBuffer,0,tmpChunkLength); + func(params...); + finishStream(); + initInputStream(tmpBuffer,0,tmpChunkLength,false); + initOutputStream(rawData,dataOffset-_rawOffset,rawChunkLength); + unRLE(false); + } catch (const ShortInputError &) { + // if this error happens on repeat/offset instead of char, though luck :( + // missing last char on src we can fix :) + initInputStream(tmpBuffer,0,tmpChunkLength,false); + initOutputStream(rawData,dataOffset-_rawOffset,rawChunkLength); + uint32_t missingNo=unRLE(true); + if (missingNo) + { + uint32_t protoSum=checksum(&rawData[dataOffset-_rawOffset],rawChunkLength-missingNo); + uint32_t fileSum=_packedData.readBE16(packedOffset+14); + uint8_t ch=((fileSum>=protoSum)?fileSum-protoSum:(0x10000U+fileSum-protoSum))/missingNo; + for (uint32_t i=0;i<missingNo;i++) outputStream.writeByte(ch); + } else throw DecompressionError(); + } + } else { + try + { + initOutputStream(rawData,dataOffset-_rawOffset,rawChunkLength); + func(params...); + } catch (const ShortInputError &) { + // same deal + if (outputStream.getOffset()+1!=rawChunkLength || _isObsfuscated) throw DecompressionError(); + uint32_t protoSum=checksum(&rawData[dataOffset-_rawOffset],rawChunkLength-1); + uint32_t fileSum=_packedData.readBE16(packedOffset+14); + uint8_t ch=fileSum-protoSum; + outputStream.writeByte(ch); + } + } + finishStream(); + }; + + auto processBlockCode=[&](bool doRLE,auto func,auto&&... params) + { + if (!_isObsfuscated || trackNo!=_minTrack) return processBlock(doRLE,func,params...); + + // fast try + if (!trackNo && restartPosition<0x10000U) for (;restartPosition<0x10000U;restartPosition++) + { + try + { + doInitContext=true; + inputUnObsfuscator.setCode(restartPosition); + limitedDecompress=8; + processBlock(doRLE,func,params...); + if ((rawData.readBE32(0)&0xffff'ff00U)!=FourCC("DOS\0")) continue; + + // now see if the candidate is any good + doInitContext=true; + inputUnObsfuscator.setCode(restartPosition); + limitedDecompress=~0U; + processBlock(doRLE,func,params...); + if (checksum(&rawData[dataOffset-_rawOffset],rawChunkLength)!=_packedData.readBE16(packedOffset+14)) continue; + return; + } catch (const Buffer::Error &) { + // just continue + } catch (const Decompressor::Error &) { + // just continue + } + } + + // slow round + limitedDecompress=~0U; + for (;restartPosition<0x20000U;restartPosition++) + { + try + { + doInitContext=true; + inputUnObsfuscator.setCode(restartPosition); + processBlock(doRLE,func,params...); + if (checksum(&rawData[dataOffset-_rawOffset],rawChunkLength)!=_packedData.readBE16(packedOffset+14)) continue; + return; + } catch (const Buffer::Error &) { + // just continue + } catch (const Decompressor::Error &) { + // just continue + } + } + throw DecompressionError(); + }; + + switch (mode) + { + case 0: + processBlockCode(false,unpackNone); + rawChunkLength=packedChunkLength; + break; + + case 1: + processBlockCode(false,unRLE,false); + break; + + case 2: + processBlockCode(true,unpackQuick); + break; + + case 3: + processBlockCode(true,unpackMedium); + break; + + case 4: + processBlockCode(true,unpackDeep); + break; + + // heavy flags: + // 2: (re-)initialize/read tables + // 4: do RLE + // heavy1 uses 4k dictionary (mode 5), whereas heavy2 uses 8k dictionary + case 5: + case 6: + processBlockCode(flags&4,unpackHeavy,flags&2,mode==6); + break; + + default: + throw DecompressionError(); + } + if (!(flags&1)) doInitContext=true; + + if (verify && checksum(&rawData[dataOffset-_rawOffset],rawChunkLength)!=_packedData.readBE16(packedOffset+14)) + throw VerificationError(); + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/DMSDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DMSDecompressor.hpp new file mode 100644 index 00000000..eb24ad2d --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DMSDecompressor.hpp @@ -0,0 +1,56 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef DMSDECOMPRESSOR_HPP +#define DMSDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +#include "common/MemoryBuffer.hpp" + +namespace ancient::internal +{ + + +class DMSDecompressor : public Decompressor +{ +public: + DMSDecompressor(const Buffer &packedData,bool verify); + + virtual ~DMSDecompressor(); + + virtual const std::string &getName() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + virtual size_t getRawSize() const noexcept override final; + + virtual size_t getImageSize() const noexcept override final; + virtual size_t getImageOffset() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + + static bool detectHeader(uint32_t hdr) noexcept; + static std::shared_ptr<Decompressor> create(const Buffer &packedData,bool exactSizeKnown,bool verify); + +private: + void decompressImpl(Buffer &rawData,bool verify,uint32_t &restartPosition); + + class ShortInputError : public Error + { + // nothing needed + }; + + const Buffer &_packedData; + + uint32_t _packedSize=0; + uint32_t _rawSize=0; + uint32_t _contextBufferSize=0; + uint32_t _tmpBufferSize=0; + uint32_t _imageSize; + uint32_t _rawOffset; + uint32_t _minTrack; + bool _isHD; + bool _isObsfuscated; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Decompressor.cpp new file mode 100644 index 00000000..6f0b976c --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Decompressor.cpp @@ -0,0 +1,108 @@ +/* Copyright (C) Teemu Suutari */ + +#include "Decompressor.hpp" + +#include <memory> +#include <vector> + +#include "BZIP2Decompressor.hpp" +#include "CRMDecompressor.hpp" +#include "DEFLATEDecompressor.hpp" +#include "DMSDecompressor.hpp" +#include "IMPDecompressor.hpp" +#include "MMCMPDecompressor.hpp" +#include "PPDecompressor.hpp" +#include "RNCDecompressor.hpp" +#include "StoneCrackerDecompressor.hpp" +#include "TPWMDecompressor.hpp" +#include "XPKMain.hpp" + +namespace ancient::internal +{ + +// --- + +static std::vector<std::pair<bool(*)(uint32_t),std::shared_ptr<Decompressor>(*)(const Buffer&,bool,bool)>> decompressors={ + {BZIP2Decompressor::detectHeader,BZIP2Decompressor::create}, + {CRMDecompressor::detectHeader,CRMDecompressor::create}, + {DEFLATEDecompressor::detectHeader,DEFLATEDecompressor::create}, + {DMSDecompressor::detectHeader,DMSDecompressor::create}, + {IMPDecompressor::detectHeader,IMPDecompressor::create}, + {MMCMPDecompressor::detectHeader,MMCMPDecompressor::create}, + {PPDecompressor::detectHeader,PPDecompressor::create}, + {RNCDecompressor::detectHeader,RNCDecompressor::create}, + {StoneCrackerDecompressor::detectHeader,StoneCrackerDecompressor::create}, + {TPWMDecompressor::detectHeader,TPWMDecompressor::create}, + {XPKMain::detectHeader,XPKMain::create}}; + +Decompressor::Decompressor() noexcept +{ + // nothing needed +} + +Decompressor::~Decompressor() +{ + // nothing needed +} + +std::shared_ptr<Decompressor> Decompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) +{ + try + { + uint32_t hdr=packedData.readBE32(0); + for (auto &it : decompressors) + { + if (it.first(hdr)) return it.second(packedData,exactSizeKnown,verify); + } + throw InvalidFormatError(); + } catch (const Buffer::Error&) { + throw InvalidFormatError(); + } +} + +bool Decompressor::detect(const Buffer &packedData) noexcept +{ + try + { + uint32_t hdr=packedData.readBE32(0); + for (auto &it : decompressors) + if (it.first(hdr)) return true; + return false; + } catch (const Buffer::Error&) { + return false; + } +} + +void Decompressor::decompress(Buffer &rawData,bool verify) +{ + // Simplifying the implementation of sub-decompressors. Just let the buffer-exception pass here, + // and that will get translated into Decompressor exceptions + try + { + decompressImpl(rawData,verify); + } catch (const Buffer::Error&) { + throw DecompressionError(); + } +} + +size_t Decompressor::getImageSize() const noexcept +{ + return 0; +} + +size_t Decompressor::getImageOffset() const noexcept +{ + return 0; +} + +size_t Decompressor::getMaxPackedSize() noexcept +{ + return 0x100'0000U; +} + +size_t Decompressor::getMaxRawSize() noexcept +{ + return 0x100'0000U; +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Decompressor.hpp new file mode 100644 index 00000000..fd0af8bd --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Decompressor.hpp @@ -0,0 +1,86 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef DECOMPRESSOR_HPP +#define DECOMPRESSOR_HPP + +#include <cstddef> +#include <cstdint> + +#include <memory> +#include <string> + +#include "common/Buffer.hpp" +#include "ancient.hpp" + +namespace ancient::internal +{ + +class Decompressor +{ +protected: + Decompressor() noexcept; + +public: + + using Error = ancient::Error; + using InvalidFormatError = ancient::InvalidFormatError; + using DecompressionError = ancient::DecompressionError; + using VerificationError = ancient::VerificationError; + + Decompressor(const Decompressor&)=delete; + Decompressor& operator=(const Decompressor&)=delete; + + virtual ~Decompressor(); + + // Name returned is human readable long name + virtual const std::string &getName() const noexcept=0; + + // PackedSize or RawSize are taken from the stream if available, 0 otherwise. + // for those compressors having 0 sizes, running decompression will update + // the values. (make sure to allocate big-enough buffer for decompression) + // There are exceptions: Some decompressors need to exact size of the packed data + // in order to decompress. For those providing a indefinitely size packed stream + // will not work + // use the "exactSizeKnown" flag for create to tell whether you know the size or not + virtual size_t getPackedSize() const noexcept=0; + virtual size_t getRawSize() const noexcept=0; + + // Actual decompression. + // verify checksum if verify==true + // can throw DecompressionError if stream cant be unpacked + // can throw VerificationError if verify enabled and checksum does not match + void decompress(Buffer &rawData,bool verify); + + // in case of disk image based formats the data does not necessarily start + // from logical beginnig of the image but it is offsetted inside the logical image + // (f.e. DMS). getDataOffset will return the offset (or 0 if not relevant or if offset does not exist) + // getImageSize will return the size of the the logical image, or 0 if not image-based format + virtual size_t getImageSize() const noexcept; + virtual size_t getImageOffset() const noexcept; + + // the functions are there to protect against "accidental" large files when parsing headers + // a.k.a. 16M should be enough for everybody (sizes do not have to accurate i.e. + // compressors can exclude header content for simplification) + // This entirely ok for the context of "old computers" and their files, + // for other usages these need to be tuned up + static size_t getMaxPackedSize() noexcept; + static size_t getMaxRawSize() noexcept; + + // Main entrypoint + // if verify=true then check the packedData for errors: CRC or other checksum if available + // check exactSizeKnown from size documentation + // can throw InvalidFormatError if stream is not recognized or it is invalid + // can throw VerificationError if verify enabled and checksum does not match + static std::shared_ptr<Decompressor> create(const Buffer &packedData,bool exactSizeKnown,bool verify); + + // Detect signature whether it matches to any known compressor + // This does not guarantee the data is decompressable though, only signature is read + static bool detect(const Buffer &packedData) noexcept; + +protected: + virtual void decompressImpl(Buffer &rawData,bool verify)=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/DynamicHuffmanDecoder.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DynamicHuffmanDecoder.hpp new file mode 100644 index 00000000..41936d5e --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/DynamicHuffmanDecoder.hpp @@ -0,0 +1,226 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef DYNAMICHUFFMANDECODER_HPP +#define DYNAMICHUFFMANDECODER_HPP + +#include <cstddef> +#include <cstdint> + +// For exception +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +template<uint32_t maxCount> +class DynamicHuffmanDecoder +{ +public: + DynamicHuffmanDecoder(uint32_t initialCount=maxCount) : + _initialCount(initialCount) + { + if (_initialCount>maxCount) throw Decompressor::DecompressionError(); + reset(); + } + + ~DynamicHuffmanDecoder() + { + // nothing needed + } + + void reset() + { + _count=_initialCount; + if (!_count) return; + for (uint32_t i=0;i<_count;i++) + { + _nodes[i].frequency=1; + _nodes[i].index=i+(maxCount-_count)*2; + _nodes[i].parent=maxCount*2-_count+(i>>1); + _nodes[i].leaves[0]=0; + _nodes[i].leaves[1]=0; + _codeMap[i+(maxCount-_count)*2]=i; + } + for (uint32_t i=maxCount*2-_count,j=0;i<maxCount*2-1;i++,j+=2) + { + uint32_t l=(j>=_count)?j+(maxCount-_count)*2:j; + uint32_t r=(j+1>=_count)?j+1+(maxCount-_count)*2:(j+1); + _nodes[i].frequency=_nodes[l].frequency+_nodes[r].frequency; + _nodes[i].index=i; + _nodes[i].parent=maxCount+(i>>1); + _nodes[i].leaves[0]=l; + _nodes[i].leaves[1]=r; + _codeMap[i]=i; + } + } + + template<typename F> + uint32_t decode(F bitReader) const + { + if (!_count) throw Decompressor::DecompressionError(); + if (_count==1) return 0; + uint32_t code=maxCount*2-2; + while (code>=maxCount) + code=_nodes[code].leaves[bitReader()?1:0]; + return code; + } + + void update(uint32_t code) + { + + if (code>=_count) throw Decompressor::DecompressionError(); + // this is a bug in LH2. Nobody else uses this codepath, so we can let it be... + if (_count==1) + { + _nodes[0].frequency=1; + return; + } + + while (code!=maxCount*2-2) + { + _nodes[code].frequency++; + + uint32_t index=_nodes[code].index; + uint32_t destIndex=index; + uint32_t freq=_nodes[code].frequency; + + while (destIndex!=maxCount*2-2 && freq>_nodes[_codeMap[destIndex+1]].frequency) destIndex++; + if (index!=destIndex) + { + auto getParentLeaf=[&](uint32_t currentCode)->uint32_t& + { + Node &parent=_nodes[_nodes[currentCode].parent]; + return parent.leaves[(parent.leaves[0]==currentCode)?0:1]; + }; + + uint32_t destCode=_codeMap[destIndex]; + std::swap(_nodes[code].index,_nodes[destCode].index); + std::swap(_codeMap[index],_codeMap[destIndex]); + std::swap(getParentLeaf(code),getParentLeaf(destCode)); + std::swap(_nodes[code].parent,_nodes[destCode].parent); + } + code=_nodes[code].parent; + } + _nodes[code].frequency++; + } + + // halve the frequencies rounding upwards + void halve() + { + if (!_count) return; + else if (_count==1) + { + _nodes[0].frequency=(_nodes[0].frequency+1)>>1; + return; + } + + for (uint32_t i=(maxCount-_count)*2,j=(maxCount-_count)*2;i<maxCount*2-1&&j<maxCount*2-_count;i++) + if (_codeMap[i]<maxCount) _nodes[_codeMap[i]].index=j++; + + for (uint32_t i=0;i<_count;i++) + { + _nodes[i].frequency=(_nodes[i].frequency+1)>>1; + _nodes[i].parent=maxCount+(_nodes[i].index>>1); + _codeMap[_nodes[i].index]=i; + } + for (uint32_t i=maxCount*2-_count,j=(maxCount-_count)*2;i<maxCount*2-1;i++,j+=2) + { + uint32_t l=_codeMap[j]; + uint32_t r=_codeMap[j+1]; + uint32_t freq=_nodes[l].frequency+_nodes[r].frequency; + _nodes[i].frequency=freq; + _nodes[i].index=i; + _nodes[i].parent=maxCount+(i>>1); + _nodes[i].leaves[0]=l; + _nodes[i].leaves[1]=r; + _codeMap[i]=i; + + for (uint32_t k=i;freq<_nodes[_codeMap[k-1]].frequency;k--) + { + uint32_t &code=_codeMap[k]; + uint32_t &destCode=_codeMap[k-1]; + std::swap(_nodes[code].index,_nodes[destCode].index); + std::swap(_nodes[code].parent,_nodes[destCode].parent); + std::swap(code,destCode); + } + } + } + + // Defined as in LH2 + void addCode() + { + if (_count>=maxCount) throw Decompressor::DecompressionError(); + uint32_t newIndex=(maxCount-_count-1)*2; + if (!_count) + { + _nodes[0].frequency=0; + _nodes[0].index=newIndex-1; + _nodes[0].parent=maxCount*2-2; + _nodes[0].leaves[0]=0; + _nodes[0].leaves[1]=0; + _codeMap[newIndex-1]=0; + _count++; + } else { + _nodes[_count].frequency=0; + _nodes[_count].index=newIndex; + _nodes[_count].parent=maxCount*2-_count-1; + _nodes[_count].leaves[0]=0; + _nodes[_count].leaves[1]=0; + _codeMap[newIndex]=_count; + + uint32_t insertIndex=newIndex+2; + + uint32_t repNode; + uint32_t parentNode; + uint32_t insertNode=maxCount*2-_count-1; + if (_count>1) + { + _codeMap[insertIndex-1]=_codeMap[insertIndex]; + _nodes[_codeMap[insertIndex-1]].index--; + + repNode=_codeMap[(maxCount-_count)*2]; + parentNode=_nodes[repNode].parent; + _nodes[parentNode].leaves[(_nodes[parentNode].leaves[0]==repNode)?0:1]=insertNode; + _nodes[repNode].parent=insertNode; + } else { + repNode=0; + parentNode=maxCount*2-1; + } + + _nodes[insertNode].frequency=_nodes[repNode].frequency; + _nodes[insertNode].index=insertIndex; + _nodes[insertNode].parent=parentNode; + _nodes[insertNode].leaves[0]=_count; + _nodes[insertNode].leaves[1]=repNode; + _codeMap[insertIndex]=insertNode; + + Node &parent=_nodes[parentNode]; + if (_count>1 && _nodes[parent.leaves[0]].index>_nodes[parent.leaves[1]].index) + std::swap(parent.leaves[0],parent.leaves[1]); + _count++; + } + } + + uint32_t getMaxFrequency() const + { + return _nodes[maxCount*2-2].frequency; + } + +private: + struct Node + { + uint32_t frequency; + uint32_t index; + uint32_t parent; + uint32_t leaves[2]; + }; + + uint32_t _initialCount; + uint32_t _count; + Node _nodes[maxCount*2-1]; + uint32_t _codeMap[maxCount*2-1]; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/FASTDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/FASTDecompressor.cpp new file mode 100644 index 00000000..c2308c5e --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/FASTDecompressor.cpp @@ -0,0 +1,78 @@ +/* Copyright (C) Teemu Suutari */ + +#include "FASTDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool FASTDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("FAST"); +} + +std::shared_ptr<XPKDecompressor> FASTDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<FASTDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +FASTDecompressor::FASTDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +FASTDecompressor::~FASTDecompressor() +{ + // nothing needed +} + +const std::string &FASTDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-FAST: Fast LZ77 compressor"; + return name; +} + +void FASTDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream forwardInputStream(_packedData,0,_packedData.size()); + BackwardInputStream backwardInputStream(_packedData,0,_packedData.size()); + forwardInputStream.link(backwardInputStream); + backwardInputStream.link(forwardInputStream); + MSBBitReader<BackwardInputStream> bitReader(backwardInputStream); + auto readBit=[&]()->uint32_t + { + return bitReader.readBitsBE16(1); + }; + auto readByte=[&]()->uint8_t + { + return forwardInputStream.readByte(); + }; + auto readShort=[&]()->uint16_t + { + const uint8_t *buf=backwardInputStream.consume(2); + uint16_t ret=uint16_t(buf[0])<<8; + return ret|uint16_t(buf[1]); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + while (!outputStream.eof()) + { + if (!readBit()) + { + outputStream.writeByte(readByte()); + } else { + uint16_t ld=readShort(); + uint32_t count=std::min(18U-(ld&0xf),uint32_t(outputStream.getEndOffset()-outputStream.getOffset())); + uint32_t distance=ld>>4; + outputStream.copy(distance,count); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/FASTDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/FASTDecompressor.hpp new file mode 100644 index 00000000..d16f091f --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/FASTDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef FASTDECOMPRESSOR_HPP +#define FASTDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class FASTDecompressor : public XPKDecompressor +{ +public: + FASTDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~FASTDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/FBR2Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/FBR2Decompressor.cpp new file mode 100644 index 00000000..68ad2cb0 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/FBR2Decompressor.cpp @@ -0,0 +1,98 @@ +/* Copyright (C) Teemu Suutari */ + +#include "FBR2Decompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool FBR2Decompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("FBR2"); +} + +std::shared_ptr<XPKDecompressor> FBR2Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<FBR2Decompressor>(hdr,recursionLevel,packedData,state,verify); +} + +FBR2Decompressor::FBR2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError();; +} + +FBR2Decompressor::~FBR2Decompressor() +{ + // nothing needed +} + +const std::string &FBR2Decompressor::getSubName() const noexcept +{ + static std::string name="XPK-FBR2: FBR2 CyberYAFA compressor"; + return name; +} + +void FBR2Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + uint8_t mode=inputStream.readByte(); + while (!outputStream.eof()) + { + bool doCopy=false; + uint32_t count=0; + switch (mode) + { + case 33: + count=uint32_t(inputStream.readByte())<<24; + count|=uint32_t(inputStream.readByte())<<16; + count|=uint32_t(inputStream.readByte())<<8; + count|=uint32_t(inputStream.readByte()); + if (count>=0x8000'0000) + { + doCopy=true; + count=0-count; + } + break; + + case 67: + count=uint32_t(inputStream.readByte())<<8; + count|=uint32_t(inputStream.readByte()); + if (count>=0x8000) + { + doCopy=true; + count=0x10000-count; + } + break; + + case 100: + count=uint32_t(inputStream.readByte()); + if (count>=0x80) + { + doCopy=true; + count=0x100-count; + } + break; + + default: + throw Decompressor::DecompressionError(); + } + + count++; + if (doCopy) { + for (uint32_t i=0;i<count;i++) outputStream.writeByte(inputStream.readByte()); + } else { + uint8_t repeatChar=inputStream.readByte(); + for (uint32_t i=0;i<count;i++) outputStream.writeByte(repeatChar); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/FBR2Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/FBR2Decompressor.hpp new file mode 100644 index 00000000..e2fb0ca7 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/FBR2Decompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef FBR2DECOMPRESSOR_HPP +#define FBR2DECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class FBR2Decompressor : public XPKDecompressor +{ +public: + FBR2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~FBR2Decompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/FRLEDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/FRLEDecompressor.cpp new file mode 100644 index 00000000..43639dfc --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/FRLEDecompressor.cpp @@ -0,0 +1,67 @@ +/* Copyright (C) Teemu Suutari */ + +#include "FRLEDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool FRLEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("FRLE"); +} + +std::shared_ptr<XPKDecompressor> FRLEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<FRLEDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +FRLEDecompressor::FRLEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +FRLEDecompressor::~FRLEDecompressor() +{ + // nothing needed +} + +const std::string &FRLEDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-FRLE: RLE-compressor"; + return name; +} + +void FRLEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + while (!outputStream.eof()) + { + auto countMod=[](uint32_t count)->uint32_t + { + return (32-(count&0x1f))+(count&0x60); + }; + + uint32_t count=uint32_t(inputStream.readByte()); + + if (count<128) + { + count=countMod(count); + for (uint32_t i=0;i<count;i++) outputStream.writeByte(inputStream.readByte()); + } else { + count=countMod(count)+1; + uint8_t ch=inputStream.readByte(); + for (uint32_t i=0;i<count;i++) outputStream.writeByte(ch); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/FRLEDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/FRLEDecompressor.hpp new file mode 100644 index 00000000..f65675a1 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/FRLEDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef FRLEDECOMPRESSOR_HPP +#define FRLEDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class FRLEDecompressor : public XPKDecompressor +{ +public: + FRLEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~FRLEDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool override) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/HFMNDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/HFMNDecompressor.cpp new file mode 100644 index 00000000..ce72f075 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/HFMNDecompressor.cpp @@ -0,0 +1,91 @@ +/* Copyright (C) Teemu Suutari */ + +#include "HFMNDecompressor.hpp" +#include "HuffmanDecoder.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/OverflowCheck.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool HFMNDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("HFMN"); +} + +std::shared_ptr<XPKDecompressor> HFMNDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<HFMNDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +HFMNDecompressor::HFMNDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr) || packedData.size()<4) + throw Decompressor::InvalidFormatError(); + uint16_t tmp=packedData.readBE16(0); + if (tmp&3U) throw Decompressor::InvalidFormatError(); // header is being written in 4 byte chunks + _headerSize=tmp&0x1ffU; // the top 7 bits are flags. No definition what they are and they are ignored in decoder... + if (OverflowCheck::sum(_headerSize,4U)>packedData.size()) throw Decompressor::InvalidFormatError(); + _rawSize=packedData.readBE16(_headerSize+2U); + if (!_rawSize) throw Decompressor::InvalidFormatError(); + _headerSize+=4; +} + +HFMNDecompressor::~HFMNDecompressor() +{ + // nothing needed +} + +const std::string &HFMNDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-HFMN: Huffman compressor"; + return name; +} + +void HFMNDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); + ForwardInputStream inputStream(_packedData,2,_headerSize); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + HuffmanDecoder<uint32_t> decoder; + uint32_t code=1; + uint32_t codeBits=1; + for (;;) + { + if (!readBit()) + { + uint32_t lit=0; + for (uint32_t i=0;i<8;i++) lit|=readBit()<<i; + decoder.insert(HuffmanCode<uint32_t>{codeBits,code,lit}); + while (!(code&1) && codeBits) + { + codeBits--; + code>>=1; + } + if (!codeBits) break; + code--; + } else { + code=(code<<1)+1; + codeBits++; + } + } + inputStream=ForwardInputStream(_packedData,_headerSize,_packedData.size()); + bitReader.reset(); + + while (!outputStream.eof()) + outputStream.writeByte(decoder.decode(readBit)); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/HFMNDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/HFMNDecompressor.hpp new file mode 100644 index 00000000..76f1b4ba --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/HFMNDecompressor.hpp @@ -0,0 +1,34 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef HFMNDECOMPRESSOR_HPP +#define HFMNDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class HFMNDecompressor : public XPKDecompressor +{ +public: + HFMNDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~HFMNDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + size_t _headerSize; + size_t _rawSize; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/HUFFDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/HUFFDecompressor.cpp new file mode 100644 index 00000000..6daa632b --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/HUFFDecompressor.cpp @@ -0,0 +1,84 @@ +/* Copyright (C) Teemu Suutari */ + +#include "HUFFDecompressor.hpp" +#include "HuffmanDecoder.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool HUFFDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("HUFF"); +} + +std::shared_ptr<XPKDecompressor> HUFFDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<HUFFDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +HUFFDecompressor::HUFFDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr) || packedData.size()<6) + throw Decompressor::InvalidFormatError(); + // version: only 0 is defined + uint16_t ver=packedData.readBE16(0); + if (ver) throw Decompressor::InvalidFormatError(); + // password: we do not support it... + uint32_t pwd=packedData.readBE32(2); + if (pwd!=0xabadcafe) throw Decompressor::InvalidFormatError(); +} + +HUFFDecompressor::~HUFFDecompressor() +{ + // nothing needed +} + +const std::string &HUFFDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-HUFF: Huffman compressor"; + return name; +} + +void HUFFDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,6,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + HuffmanDecoder<uint32_t> decoder; + for (uint32_t i=0;i<256;i++) + { + uint8_t codeBits=readByte()+1; + if (!codeBits) continue; + if (codeBits>32) throw Decompressor::DecompressionError(); + uint32_t code=0; + int32_t shift=-codeBits; + for (uint32_t j=0;j<codeBits;j+=8) + { + code=(code<<8)|readByte(); + shift+=8; + } + code=(code>>shift)&((1<<codeBits)-1); + decoder.insert(HuffmanCode<uint32_t>{codeBits,code,i}); + } + + while (!outputStream.eof()) + outputStream.writeByte(decoder.decode(readBit)); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/HUFFDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/HUFFDecompressor.hpp new file mode 100644 index 00000000..fba47bd6 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/HUFFDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef HUFFDECOMPRESSOR_HPP +#define HUFFDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class HUFFDecompressor : public XPKDecompressor +{ +public: + HUFFDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~HUFFDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/HuffmanDecoder.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/HuffmanDecoder.hpp new file mode 100644 index 00000000..e47649ba --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/HuffmanDecoder.hpp @@ -0,0 +1,228 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef HUFFMANDECODER_HPP +#define HUFFMANDECODER_HPP + +#include <cstddef> +#include <cstdint> + +#include <vector> +#include <utility> + +// For exception +#include "Decompressor.hpp" + +#include "common/MemoryBuffer.hpp" + +namespace ancient::internal +{ + +template<typename T> +struct HuffmanCode +{ + uint32_t length; + uint32_t code; + + T value; +}; + +template<typename T> class OptionalHuffmanDecoder; + +template<typename T> +class HuffmanDecoder +{ +friend class OptionalHuffmanDecoder<T>; +private: + struct Node + { + uint32_t sub[2]; + T value; + + Node(uint32_t _sub0,uint32_t _sub1,T _value) : + sub{_sub0,_sub1}, + value(_value) + { + // nothing needed + } + + Node(Node &&source) : + sub{source.sub[0],source.sub[1]}, + value(source.value) + { + // nothing needed + } + + Node& operator=(Node &&source) + { + if (this!=&source) + { + sub[0]=source.sub[0]; + sub[1]=source.sub[1]; + value=source.value; + } + return *this; + } + }; + +public: + HuffmanDecoder() + { + // nothing needed + } + + template<typename ...Args> + HuffmanDecoder(const Args&& ...args) : + HuffmanDecoder() + { + const HuffmanCode<T> list[sizeof...(args)]={args...}; + for (auto &item : list) + insert(item); + } + + ~HuffmanDecoder() + { + } + + void reset() + { + _table.clear(); + } + + template<typename F> + const T &decode(F bitReader) const + { + if (!_table.size()) throw Decompressor::DecompressionError(); + uint32_t i=0; + while (_table[i].sub[0] || _table[i].sub[1]) + { + i=_table[i].sub[bitReader()?1:0]; + if (!i) throw Decompressor::DecompressionError(); + } + return _table[i].value; + } + + void insert(const HuffmanCode<T> &code) + { + uint32_t i=0,length=uint32_t(_table.size()); + for (int32_t currentBit=code.length;currentBit>=0;currentBit--) + { + uint32_t codeBit=(currentBit && ((code.code>>(currentBit-1U))&1U))?1U:0; + if (i!=length) + { + if (!currentBit || (!_table[i].sub[0] && !_table[i].sub[1])) throw Decompressor::DecompressionError(); + uint32_t &tmp=_table[i].sub[codeBit]; + if (!tmp) tmp=i=length; + else i=tmp; + } else { + _table.emplace_back((currentBit&&!codeBit)?length+1:0,(currentBit&&codeBit)?length+1:0,currentBit?T():code.value); + length++; + i++; + } + } + } + + // create orderly Huffman table, as used by Deflate and Bzip2 + void createOrderlyHuffmanTable(const uint8_t *bitLengths,uint32_t bitTableLength) + { + uint8_t minDepth=32,maxDepth=0; + // some optimization: more tables + uint16_t firstIndex[33],lastIndex[33]; + MemoryBuffer nextIndexBuffer(bitTableLength*sizeof(uint16_t)); + uint16_t *nextIndex=nextIndexBuffer.cast<uint16_t>(); + for (uint32_t i=1;i<33;i++) + firstIndex[i]=0xffffU; + + uint32_t realItems=0; + for (uint32_t i=0;i<bitTableLength;i++) + { + uint8_t length=bitLengths[i]; + if (length>32) throw Decompressor::DecompressionError(); + if (length) + { + if (length<minDepth) minDepth=length; + if (length>maxDepth) maxDepth=length; + if (firstIndex[length]==0xffffU) + { + firstIndex[length]=i; + lastIndex[length]=i; + } else { + nextIndex[lastIndex[length]]=i; + lastIndex[length]=i; + } + realItems++; + } + } + if (!maxDepth) throw Decompressor::DecompressionError(); + // optimization, the multiple depends how sparse the tree really is. (minimum is *2) + // usually it is sparse. + _table.reserve(realItems*3); + + uint32_t code=0; + for (uint32_t depth=minDepth;depth<=maxDepth;depth++) + { + if (firstIndex[depth]!=0xffffU) + nextIndex[lastIndex[depth]]=bitTableLength; + + for (uint32_t i=firstIndex[depth];i<bitTableLength;i=nextIndex[i]) + { + insert(HuffmanCode<T>{depth,code>>(maxDepth-depth),(T)i}); + code+=1<<(maxDepth-depth); + } + } + } + +private: + std::vector<Node> _table; +}; + +template<typename T> +class OptionalHuffmanDecoder +{ +public: + OptionalHuffmanDecoder() : + _base() + { + // nothing needed + } + + ~OptionalHuffmanDecoder() + { + // nothing needed + } + + void reset() + { + _base.reset(); + } + + void setEmpty(T value) + { + reset(); + _emptyValue=value; + } + + template<typename F> + T decode(F bitReader) const + { + if (!_base._table.size()) return _emptyValue; + else return _base.decode(bitReader); + } + + void insert(const HuffmanCode<T> &code) + { + _base.insert(code); + } + + void createOrderlyHuffmanTable(const uint8_t *bitLengths,uint32_t bitTableLength) + { + _base.createOrderlyHuffmanTable(bitLengths,bitTableLength); + } + +private: + HuffmanDecoder<T> _base; + T _emptyValue=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/ILZRDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ILZRDecompressor.cpp new file mode 100644 index 00000000..b6fb93ed --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ILZRDecompressor.cpp @@ -0,0 +1,73 @@ +/* Copyright (C) Teemu Suutari */ + +#include "ILZRDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool ILZRDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("ILZR"); +} + +std::shared_ptr<XPKDecompressor> ILZRDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<ILZRDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +ILZRDecompressor::ILZRDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr) || packedData.size()<2) + throw Decompressor::InvalidFormatError(); + _rawSize=_packedData.readBE16(0); + if (!_rawSize) throw Decompressor::InvalidFormatError(); +} + +ILZRDecompressor::~ILZRDecompressor() +{ + // nothing needed +} + +const std::string &ILZRDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-ILZR: Incremental Lempel-Ziv-Renau compressor"; + return name; +} + +void ILZRDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); + + ForwardInputStream inputStream(_packedData,2,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + uint32_t bits=8; + while (!outputStream.eof()) + { + if (readBits(1)) + { + outputStream.writeByte(readBits(8)); + } else { + while (outputStream.getOffset()>(1ULL<<bits)) bits++; + uint32_t position=readBits(bits); + uint32_t count=readBits(4)+3; + + if (position>=outputStream.getOffset()) throw Decompressor::DecompressionError(); + outputStream.copy(outputStream.getOffset()-position,count); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/ILZRDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ILZRDecompressor.hpp new file mode 100644 index 00000000..2dbdf042 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ILZRDecompressor.hpp @@ -0,0 +1,33 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef ILZRDECOMPRESSOR_HPP +#define ILZRDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class ILZRDecompressor : public XPKDecompressor +{ +public: + ILZRDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~ILZRDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + size_t _rawSize=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/IMPDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/IMPDecompressor.cpp new file mode 100644 index 00000000..991f1d36 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/IMPDecompressor.cpp @@ -0,0 +1,294 @@ +/* Copyright (C) Teemu Suutari */ + +#include "IMPDecompressor.hpp" +#include "HuffmanDecoder.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" +#include "common/OverflowCheck.hpp" + + +namespace ancient::internal +{ + +static bool readIMPHeader(uint32_t hdr,uint32_t &addition) noexcept +{ + switch (hdr) + { + case FourCC("ATN!"): + case FourCC("EDAM"): + case FourCC("IMP!"): + case FourCC("M.H."): + addition=7; + return true; + + case FourCC("BDPI"): + addition=0x6e8; + return true; + + case FourCC("CHFI"): + addition=0xfe4; + return true; + + case FourCC("RDC9"): // Files do not contain checksum + + // I haven't got these files to be sure what is the addition + case FourCC("Dupa"): + case FourCC("FLT!"): + case FourCC("PARA"): + addition=0; // disable checksum for now + return true; + + default: + return false; + } +} + +bool IMPDecompressor::detectHeader(uint32_t hdr) noexcept +{ + uint32_t dummy; + return readIMPHeader(hdr,dummy); +} + +bool IMPDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("IMPL"); +} + +std::shared_ptr<Decompressor> IMPDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) +{ + return std::make_shared<IMPDecompressor>(packedData,verify); +} + +std::shared_ptr<XPKDecompressor> IMPDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<IMPDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +IMPDecompressor::IMPDecompressor(const Buffer &packedData,bool verify) : + _packedData(packedData) +{ + uint32_t hdr=packedData.readBE32(0); + uint32_t checksumAddition; + if (!readIMPHeader(hdr,checksumAddition) || packedData.size()<0x32) throw InvalidFormatError(); + + _rawSize=packedData.readBE32(4); + _endOffset=packedData.readBE32(8); + if ((_endOffset&1) || _endOffset<0xc || _endOffset+0x32>packedData.size() || + !_rawSize || !_endOffset || + _rawSize>getMaxRawSize() || _endOffset>getMaxPackedSize()) throw InvalidFormatError(); + uint32_t checksum=packedData.readBE32(_endOffset+0x2e); + if (verify && checksumAddition) + { + // size is divisible by 2 + uint32_t sum=checksumAddition; + for (uint32_t i=0;i<_endOffset+0x2e;i+=2) + { + // TODO: slow, optimize + uint16_t tmp=_packedData.readBE16(i); + sum+=uint32_t(tmp); + } + if (checksum!=sum) throw InvalidFormatError(); + } +} + +IMPDecompressor::IMPDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr) || packedData.size()<0x2e) throw InvalidFormatError(); + + _rawSize=packedData.readBE32(4); + _endOffset=packedData.readBE32(8); + if ((_endOffset&1) || _endOffset<0xc || OverflowCheck::sum(_endOffset,0x2eU)>packedData.size()) throw InvalidFormatError(); + _isXPK=true; +} + +IMPDecompressor::~IMPDecompressor() +{ + // nothing needed +} + +const std::string &IMPDecompressor::getName() const noexcept +{ + static std::string name="IMP: File Imploder"; + return name; +} + +const std::string &IMPDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-IMPL: File Imploder"; + return name; +} + +size_t IMPDecompressor::getPackedSize() const noexcept +{ + return _endOffset+0x32; +} + +size_t IMPDecompressor::getRawSize() const noexcept +{ + return _rawSize; +} + +void IMPDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + if (rawData.size()<_rawSize) throw DecompressionError(); + + class IMPInputStream + { + public: + IMPInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset) : + _bufPtr(buffer.data()), + _currentOffset(endOffset), + _endOffset(startOffset), + _refOffset(endOffset) + { + if (_currentOffset<_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); + uint8_t markerByte=buffer.read8(_currentOffset+16); + if (!(markerByte&0x80)) + { + if (_currentOffset==_endOffset) throw Decompressor::DecompressionError(); + _currentOffset--; + } + } + + ~IMPInputStream() + { + // nothing needed + } + + uint8_t readByte() + { + // streamreader with funny ordering + auto sourceOffset=[&](size_t i)->size_t + { + if (i>=12) + { + return i; + } else { + if (i<4) + { + return i+_refOffset+8; + } else if (i<8) { + return i+_refOffset; + } else { + return i+_refOffset-8; + } + } + }; + if (_currentOffset<=_endOffset) throw Decompressor::DecompressionError(); + return _bufPtr[sourceOffset(--_currentOffset)]; + } + + bool eof() const { return _currentOffset==_endOffset; } + + private: + const uint8_t *_bufPtr; + size_t _currentOffset; + size_t _endOffset; + size_t _refOffset; + }; + + IMPInputStream inputStream(_packedData,0,_endOffset); + MSBBitReader<IMPInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + // the anchor-bit does not seem always to be at the correct place + { + uint8_t halfByte=_packedData.read8(_endOffset+17); + for (uint32_t i=0;i<7;i++) + if (halfByte&(1<<i)) + { + bitReader.reset(halfByte>>(i+1),7-i); + break; + } + } + + BackwardOutputStream outputStream(rawData,0,_rawSize); + + // tables + uint16_t distanceValues[2][4]; + for (uint32_t i=0;i<8;i++) + distanceValues[i>>2][i&3]=_packedData.readBE16(_endOffset+18+i*2); + uint8_t distanceBits[3][4]; + for (uint32_t i=0;i<12;i++) + distanceBits[i>>2][i&3]=_packedData.read8(_endOffset+34+i); + + // length, distance & literal counts are all intertwined + HuffmanDecoder<uint8_t> lldDecoder + { + HuffmanCode<uint8_t>{1,0b00000,0}, + HuffmanCode<uint8_t>{2,0b00010,1}, + HuffmanCode<uint8_t>{3,0b00110,2}, + HuffmanCode<uint8_t>{4,0b01110,3}, + HuffmanCode<uint8_t>{5,0b11110,4}, + HuffmanCode<uint8_t>{5,0b11111,5} + }; + + HuffmanDecoder<uint8_t> lldDecoder2 + { + HuffmanCode<uint8_t>{1,0b00,0}, + HuffmanCode<uint8_t>{2,0b10,1}, + HuffmanCode<uint8_t>{2,0b11,2} + }; + + // finally loop + uint32_t litLength=_packedData.readBE32(_endOffset+12); + + for (;;) + { + for (uint32_t i=0;i<litLength;i++) outputStream.writeByte(readByte()); + + if (outputStream.eof()) break; + + // now the intertwined Huffman table reads. + uint32_t i0=lldDecoder.decode(readBit); + uint32_t selector=(i0<4)?i0:3; + uint32_t count=i0+2; + if (count==6) + { + count+=readBits(3); + } else if (count==7) { + count=readByte(); + // why this is error? (Well, it just is) + if (!count) throw DecompressionError(); + } + + static const uint8_t literalLengths[4]={6,10,10,18}; + static const uint8_t literalBits[3][4]={ + {1,1,1,1}, + {2,3,3,4}, + {4,5,7,14}}; + uint32_t i1=lldDecoder2.decode(readBit); + litLength=i1+i1; + if (litLength==4) + { + litLength=literalLengths[selector]; + } + litLength+=readBits(literalBits[i1][selector]); + + uint32_t i2=lldDecoder2.decode(readBit); + uint32_t distance=1+((i2)?distanceValues[i2-1][selector]:0)+readBits(distanceBits[i2][selector]); + + outputStream.copy(distance,count); + } +} + +void IMPDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + if (_rawSize!=rawData.size()) throw DecompressionError(); + return decompressImpl(rawData,verify); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/IMPDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/IMPDecompressor.hpp new file mode 100644 index 00000000..e0af8669 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/IMPDecompressor.hpp @@ -0,0 +1,44 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef IMPDECOMPRESSOR_HPP +#define IMPDECOMPRESSOR_HPP + +#include "Decompressor.hpp" +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class IMPDecompressor : public Decompressor, public XPKDecompressor +{ +public: + IMPDecompressor(const Buffer &packedData,bool verify); + IMPDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + virtual ~IMPDecompressor(); + + virtual const std::string &getName() const noexcept override final; + virtual const std::string &getSubName() const noexcept override final; + + virtual size_t getPackedSize() const noexcept override final; + virtual size_t getRawSize() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeader(uint32_t hdr) noexcept; + static bool detectHeaderXPK(uint32_t hdr) noexcept; + + static std::shared_ptr<Decompressor> create(const Buffer &packedData,bool exactSizeKnown,bool verify); + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _rawSize=0; + uint32_t _endOffset=0; + bool _isXPK=false; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/InputStream.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/InputStream.cpp new file mode 100644 index 00000000..42657197 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/InputStream.cpp @@ -0,0 +1,113 @@ +/* Copyright (C) Teemu Suutari */ + +#include "InputStream.hpp" +// for exceptions +#include "Decompressor.hpp" +#include "common/OverflowCheck.hpp" + + +namespace ancient::internal +{ + +ForwardInputStream::ForwardInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset,bool allowOverrun) : + _bufPtr(buffer.data()), + _currentOffset(startOffset), + _endOffset(endOffset), + _allowOverrun(allowOverrun) +{ + if (_currentOffset>_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); +} + +ForwardInputStream::~ForwardInputStream() +{ + // nothing needed +} + +uint8_t ForwardInputStream::readByte() +{ + if (_currentOffset>=_endOffset) + { + if (_allowOverrun) + { + _currentOffset++; + return 0; + } + throw Decompressor::DecompressionError(); + } + uint8_t ret=_bufPtr[_currentOffset++]; + if (_linkedInputStream) _linkedInputStream->setOffset(_currentOffset); + return ret; +} + +const uint8_t *ForwardInputStream::consume(size_t bytes,uint8_t *buffer) +{ + if (OverflowCheck::sum(_currentOffset,bytes)>_endOffset) + { + if (_allowOverrun && buffer) + { + for (size_t i=0;i<bytes;i++) + { + buffer[i]=(_currentOffset<_endOffset)?_bufPtr[_currentOffset]:0; + _currentOffset++; + } + return buffer; + } + throw Decompressor::DecompressionError(); + } + const uint8_t *ret=&_bufPtr[_currentOffset]; + _currentOffset+=bytes; + if (_linkedInputStream) _linkedInputStream->setOffset(_currentOffset); + return ret; +} + +BackwardInputStream::BackwardInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset,bool allowOverrun) : + _bufPtr(buffer.data()), + _currentOffset(endOffset), + _endOffset(startOffset), + _allowOverrun(allowOverrun) +{ + if (_currentOffset<_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); +} + +BackwardInputStream::~BackwardInputStream() +{ + // nothing needed +} + +uint8_t BackwardInputStream::readByte() +{ + if (_currentOffset<=_endOffset) + { + if (_allowOverrun) + { + --_currentOffset; + return 0; + } + throw Decompressor::DecompressionError(); + } + uint8_t ret=_bufPtr[--_currentOffset]; + if (_linkedInputStream) _linkedInputStream->setOffset(_currentOffset); + return ret; +} + +const uint8_t *BackwardInputStream::consume(size_t bytes,uint8_t *buffer) +{ + if (_currentOffset<OverflowCheck::sum(_endOffset,bytes)) + { + if (_allowOverrun && buffer) + { + for (size_t i=bytes;i;i--) + { + buffer[i-1]=(_currentOffset>_endOffset)?_bufPtr[_currentOffset-1]:0; + --_currentOffset; + } + return buffer; + } + throw Decompressor::DecompressionError(); + } + _currentOffset-=bytes; + if (_linkedInputStream) _linkedInputStream->setOffset(_currentOffset); + return &_bufPtr[_currentOffset]; +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/InputStream.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/InputStream.hpp new file mode 100644 index 00000000..21019be9 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/InputStream.hpp @@ -0,0 +1,236 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef INPUTSTREAM_HPP +#define INPUTSTREAM_HPP + +#include <cstddef> +#include <cstdint> + +#include <algorithm> + +#include "common/Buffer.hpp" + +namespace ancient::internal +{ + +class BackwardInputStream; + +class ForwardInputStream +{ + friend class BackwardInputStream; + +public: + ForwardInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset,bool allowOverrun=false); + ~ForwardInputStream(); + + uint8_t readByte(); + const uint8_t *consume(size_t bytes,uint8_t *buffer=nullptr); + + bool eof() const { return _currentOffset==_endOffset; } + size_t getOffset() const { return _currentOffset; } + size_t getEndOffset() const { return _endOffset; } + void link(BackwardInputStream &stream) { _linkedInputStream=&stream; } + +private: + void setOffset(size_t offset) { _endOffset=offset; } + + const uint8_t *_bufPtr; + size_t _currentOffset; + size_t _endOffset; + bool _allowOverrun; + + BackwardInputStream *_linkedInputStream=nullptr; +}; + + +class BackwardInputStream +{ + friend class ForwardInputStream; +public: + BackwardInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset,bool allowOverrun=false); + ~BackwardInputStream(); + + uint8_t readByte(); + const uint8_t *consume(size_t bytes,uint8_t *buffer=nullptr); + + bool eof() const { return _currentOffset==_endOffset; } + size_t getOffset() const { return _currentOffset; } + void link(ForwardInputStream &stream) { _linkedInputStream=&stream; } + +private: + void setOffset(size_t offset) { _endOffset=offset; } + + const uint8_t *_bufPtr; + size_t _currentOffset; + size_t _endOffset; + bool _allowOverrun; + + ForwardInputStream *_linkedInputStream=nullptr; +}; + + +template<typename T> +class LSBBitReader +{ +public: + LSBBitReader(T &inputStream) : + _inputStream(inputStream) + { + // nothing needed + } + + ~LSBBitReader() + { + // nothing needed + } + + uint32_t readBits8(uint32_t count) + { + return readBitsInternal(count,[&](){ + _bufContent=_inputStream.readByte(); + _bufLength=8; + }); + } + + uint32_t readBitsBE16(uint32_t count) + { + return readBitsInternal(count,[&](){ + uint8_t tmp[2]; + const uint8_t *buf=_inputStream.consume(2,tmp); + _bufContent=(uint32_t(buf[0])<<8)|uint32_t(buf[1]); + _bufLength=16; + }); + } + + uint32_t readBitsBE32(uint32_t count) + { + return readBitsInternal(count,[&](){ + uint8_t tmp[4]; + const uint8_t *buf=_inputStream.consume(4,tmp); + _bufContent=(uint32_t(buf[0])<<24)|(uint32_t(buf[1])<<16)| + (uint32_t(buf[2])<<8)|uint32_t(buf[3]); + _bufLength=32; + }); + } + + // RNC + uint32_t readBits16Limit(uint32_t count) + { + return readBitsInternal(count,[&](){ + _bufContent=_inputStream.readByte(); + if (_inputStream.eof()) + { + _bufLength=8; + } else { + _bufContent=_bufContent|(uint32_t(_inputStream.readByte())<<8); + _bufLength=16; + } + }); + } + + void reset(uint32_t bufContent=0,uint8_t bufLength=0) + { + _bufContent=bufContent; + _bufLength=bufLength; + } + +private: + template<typename F> + uint32_t readBitsInternal(uint32_t count,F readWord) + { + uint32_t ret=0,pos=0; + while (count) + { + if (!_bufLength) + readWord(); + uint8_t maxCount=std::min(uint8_t(count),_bufLength); + ret|=(_bufContent&((1<<maxCount)-1))<<pos; + _bufContent>>=maxCount; + _bufLength-=maxCount; + count-=maxCount; + pos+=maxCount; + } + return ret; + } + + T &_inputStream; + uint32_t _bufContent=0; + uint8_t _bufLength=0; +}; + + +template<typename T> +class MSBBitReader +{ +public: + MSBBitReader(T &inputStream) : + _inputStream(inputStream) + { + // nothing needed + } + + ~MSBBitReader() + { + // nothing needed + } + + uint32_t readBits8(uint32_t count) + { + return readBitsInternal(count,[&](){ + _bufContent=_inputStream.readByte(); + _bufLength=8; + }); + } + + uint32_t readBitsBE16(uint32_t count) + { + return readBitsInternal(count,[&](){ + uint8_t tmp[2]; + const uint8_t *buf=_inputStream.consume(2,tmp); + _bufContent=(uint32_t(buf[0])<<8)|uint32_t(buf[1]); + _bufLength=16; + }); + } + + uint32_t readBitsBE32(uint32_t count) + { + return readBitsInternal(count,[&](){ + uint8_t tmp[4]; + const uint8_t *buf=_inputStream.consume(4,tmp); + _bufContent=(uint32_t(buf[0])<<24)|(uint32_t(buf[1])<<16)| + (uint32_t(buf[2])<<8)|uint32_t(buf[3]); + _bufLength=32; + }); + } + + void reset(uint32_t bufContent=0,uint8_t bufLength=0) + { + _bufContent=bufContent; + _bufLength=bufLength; + } + +private: + template<typename F> + uint32_t readBitsInternal(uint32_t count,F readWord) + { + uint32_t ret=0; + while (count) + { + if (!_bufLength) + readWord(); + uint8_t maxCount=std::min(uint8_t(count),_bufLength); + _bufLength-=maxCount; + ret=(ret<<maxCount)|((_bufContent>>_bufLength)&((1<<maxCount)-1)); + count-=maxCount; + } + return ret; + } + + T &_inputStream; + uint32_t _bufContent=0; + uint8_t _bufLength=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LHLBDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LHLBDecompressor.cpp new file mode 100644 index 00000000..7ddd51bf --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LHLBDecompressor.cpp @@ -0,0 +1,108 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LHLBDecompressor.hpp" + +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "DynamicHuffmanDecoder.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool LHLBDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("LHLB"); +} + +std::shared_ptr<XPKDecompressor> LHLBDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<LHLBDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +LHLBDecompressor::LHLBDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +LHLBDecompressor::~LHLBDecompressor() +{ + // nothing needed +} + +const std::string &LHLBDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-LHLB: LZRW-compressor"; + return name; +} + +void LHLBDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + // Same logic as in Choloks pascal implementation + // Differences to LH1: + // - LHLB does not halve probabilities at 32k + // - 314 vs. 317 sized huffman entry + // - no end code + // - different distance/count logic + + DynamicHuffmanDecoder<317> decoder; + + while (!outputStream.eof()) + { + uint32_t code=decoder.decode(readBit); + if (code==316) break; + if (decoder.getMaxFrequency()<0x8000U) decoder.update(code); + + if (code<256) + { + outputStream.writeByte(code); + } else { + static const uint8_t distanceHighBits[256]={ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, + 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, + + 10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11, + 12,12,12,12,13,13,13,13, 14,14,14,14,15,15,15,15, + 16,16,16,16,17,17,17,17, 18,18,18,18,19,19,19,19, + 20,20,20,20,21,21,21,21, 22,22,22,22,23,23,23,23, + 24,24,25,25,26,26,27,27, 28,28,29,29,30,30,31,31, + 32,32,33,33,34,34,35,35, 36,36,37,37,38,38,39,39, + 40,40,41,41,42,42,43,43, 44,44,45,45,46,46,47,47, + 48,49,50,51,52,53,54,55, 56,57,58,59,60,61,62,63}; + static const uint8_t distanceBits[16]={1,1,2,2,2,3,3,3,3,4,4,4,5,5,5,6}; + + uint32_t tmp=readBits(8); + uint32_t distance=uint32_t(distanceHighBits[tmp])<<6; + uint32_t bits=distanceBits[tmp>>4]; + tmp=(tmp<<bits)|readBits(bits); + distance|=tmp&63; + uint32_t count=code-255; + + outputStream.copy(distance,count); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LHLBDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LHLBDecompressor.hpp new file mode 100644 index 00000000..31c6280b --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LHLBDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LHLBDECOMPRESSOR_HPP +#define LHLBDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class LHLBDecompressor : public XPKDecompressor +{ +public: + LHLBDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~LHLBDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LIN1Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LIN1Decompressor.cpp new file mode 100644 index 00000000..b2f8cd54 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LIN1Decompressor.cpp @@ -0,0 +1,123 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LIN1Decompressor.hpp" + +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool LIN1Decompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("LIN1") || hdr==FourCC("LIN3"); +} + +std::shared_ptr<XPKDecompressor> LIN1Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<LIN1Decompressor>(hdr,recursionLevel,packedData,state,verify); +} + +LIN1Decompressor::LIN1Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); + _ver=(hdr==FourCC("LIN1"))?1:3; + if (packedData.size()<5) throw Decompressor::InvalidFormatError(); + + uint32_t tmp=packedData.readBE32(0); + if (tmp) throw Decompressor::InvalidFormatError(); // password set +} + +LIN1Decompressor::~LIN1Decompressor() +{ + // nothing needed +} + +const std::string &LIN1Decompressor::getSubName() const noexcept +{ + static std::string name1="XPK-LIN1: LIN1 LINO packer"; + static std::string name3="XPK-LIN3: LIN3 LINO packer"; + return (_ver==1)?name1:name3; +} + +void LIN1Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,5,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + size_t rawSize=rawData.size(); + ForwardOutputStream outputStream(rawData,0,rawSize); + + while (!outputStream.eof()) + { + if (!readBits(1)) + { + outputStream.writeByte(readByte()^0x55); + } else { + uint32_t count=3; + if (readBits(1)) + { + count=readBits(2); + if (count==3) + { + count=readBits(3); + if (count==7) + { + count=readBits(4); + if (count==15) + { + count=readByte(); + if (count==0xff) throw Decompressor::DecompressionError(); + count+=3; + } else count+=14; + } else count+=7; + } else count+=4; + } + uint32_t distance = 0; + switch (readBits(2)) + { + case 0: + distance=readByte()+1; + break; + + case 1: + distance=uint32_t(readBits(2))<<8; + distance|=readByte(); + distance+=0x101; + break; + + case 2: + distance=uint32_t(readBits(4))<<8; + distance|=readByte(); + distance+=0x501; + break; + + case 3: + distance=uint32_t(readBits(6))<<8; + distance|=readByte(); + distance+=0x1501; + break; + } + + // buggy compressors + count=std::min(count,uint32_t(rawSize-outputStream.getOffset())); + if (!count) throw Decompressor::DecompressionError(); + + outputStream.copy(distance,count); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LIN1Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LIN1Decompressor.hpp new file mode 100644 index 00000000..ed1967f8 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LIN1Decompressor.hpp @@ -0,0 +1,33 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LIN1DECOMPRESSOR_HPP +#define LIN1DECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class LIN1Decompressor : public XPKDecompressor +{ +public: + LIN1Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~LIN1Decompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _ver=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LIN2Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LIN2Decompressor.cpp new file mode 100644 index 00000000..d69b65bf --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LIN2Decompressor.cpp @@ -0,0 +1,224 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LIN2Decompressor.hpp" +#include "HuffmanDecoder.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" +#include "common/OverflowCheck.hpp" + + +namespace ancient::internal +{ + +bool LIN2Decompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("LIN2") || hdr==FourCC("LIN4"); +} + +std::shared_ptr<XPKDecompressor> LIN2Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<LIN2Decompressor>(hdr,recursionLevel,packedData,state,verify); +} + +LIN2Decompressor::LIN2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); + _ver=(hdr==FourCC("LIN2"))?2:4; + if (packedData.size()<10) throw Decompressor::InvalidFormatError(); + + uint32_t tmp=packedData.readBE32(0); + if (tmp) throw Decompressor::InvalidFormatError(); // password set + + // LIN4 is very similar to LIN2 - it only has 5 bit literals instead of 4 bit literals + // (and thus larger table at the end of the stream) + // Also, the huffman decoder for length is different + + _endStreamOffset=packedData.size()-1; + const uint8_t *bufPtr=_packedData.data(); + while (_endStreamOffset && bufPtr[--_endStreamOffset]!=0xffU); + // end stream + // 1 byte, byte before 0xff + // 0x10 bytes/0x20 for table + if (_endStreamOffset<0x11+0xa) throw Decompressor::InvalidFormatError(); + _endStreamOffset-=(_ver==2)?0x11:0x21; + + size_t midStreamOffset=(_ver==2)?0x16:0x26; + // midstream + // from endstream without + // add 0x10/0x20 byte back to point after table + // add 6 bytes to point to correct place + + tmp=packedData.readBE32(4); + if (OverflowCheck::sum(_endStreamOffset,midStreamOffset)<OverflowCheck::sum(tmp,10U) || tmp<midStreamOffset) throw Decompressor::InvalidFormatError(); + _midStreamOffset=_endStreamOffset-tmp+midStreamOffset; +} + +LIN2Decompressor::~LIN2Decompressor() +{ + // nothing needed +} + +const std::string &LIN2Decompressor::getSubName() const noexcept +{ + static std::string name2="XPK-LIN2: LIN2 LINO packer"; + static std::string name4="XPK-LIN4: LIN4 LINO packer"; + return (_ver==2)?name2:name4; +} + +void LIN2Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + // three streams. + // 1. ordinary bit stream out of words (readBits) + // 2. bit stream for literals (readBit) + // 3. nibble stream for literal (read4Bits) + // at the end of the stream there is a literal table of 16/32 bytes + // apart from confusing naming, there are also some nasty + // interdependencies :( + ForwardInputStream forwardInputStream(_packedData,10,_midStreamOffset); + ForwardInputStream middleInputStream(_packedData,_midStreamOffset,_endStreamOffset); + BackwardInputStream backwardInputStream(_packedData,_midStreamOffset,_endStreamOffset); + middleInputStream.link(backwardInputStream); + backwardInputStream.link(middleInputStream); + MSBBitReader<ForwardInputStream> bitsReader(forwardInputStream); + MSBBitReader<ForwardInputStream> bitReader(middleInputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitsReader.readBits8(count); + }; + + { + uint8_t tmp=middleInputStream.readByte(); + if (tmp>8) throw Decompressor::DecompressionError(); + bitReader.reset(middleInputStream.readByte()>>tmp,8-tmp); + } + auto readBit=[&]()->uint8_t + { + return bitReader.readBits8(1); + }; + + bool buf4Incomplete=false; + uint8_t nibbleContent=0; + { + uint8_t tmp=_packedData.read8(9); + buf4Incomplete=!!tmp; + if (buf4Incomplete) + nibbleContent=backwardInputStream.readByte(); + } + // this is a rather strange thing... + auto read4Bits=[&](bool fullByte)->uint8_t + { + if (!fullByte) + { + buf4Incomplete=!buf4Incomplete; + if (!buf4Incomplete) + { + return nibbleContent&0xf; + } else { + nibbleContent=backwardInputStream.readByte(); + return nibbleContent>>4; + } + } else { + if (buf4Incomplete) + { + uint8_t ret=nibbleContent&0xf; + nibbleContent=backwardInputStream.readByte(); + ret|=nibbleContent&0xf0U; + return ret; + } else { + return backwardInputStream.readByte(); + } + } + }; + + const uint8_t *literalTable=&_packedData[_endStreamOffset]; + + size_t rawSize=rawData.size(); + ForwardOutputStream outputStream(rawData,0,rawSize); + + // little meh to initialize both (intentionally deleted copy/assign) + HuffmanDecoder<uint8_t> lengthDecoder2 + { + HuffmanCode<uint8_t>{1,0b000000,3}, + HuffmanCode<uint8_t>{3,0b000100,4}, + HuffmanCode<uint8_t>{3,0b000101,5}, + HuffmanCode<uint8_t>{3,0b000110,6}, + HuffmanCode<uint8_t>{6,0b111000,7}, + HuffmanCode<uint8_t>{6,0b111001,8}, + HuffmanCode<uint8_t>{6,0b111010,9}, + HuffmanCode<uint8_t>{6,0b111011,10}, + HuffmanCode<uint8_t>{6,0b111100,11}, + HuffmanCode<uint8_t>{6,0b111101,12}, + HuffmanCode<uint8_t>{6,0b111110,13}, + HuffmanCode<uint8_t>{6,0b111111,0} + }; + + HuffmanDecoder<uint8_t> lengthDecoder4 + { + HuffmanCode<uint8_t>{2,0b0000000,3}, + HuffmanCode<uint8_t>{2,0b0000001,4}, + HuffmanCode<uint8_t>{2,0b0000010,5}, + HuffmanCode<uint8_t>{4,0b0001100,6}, + HuffmanCode<uint8_t>{4,0b0001101,7}, + HuffmanCode<uint8_t>{4,0b0001110,8}, + HuffmanCode<uint8_t>{7,0b1111000,9}, + HuffmanCode<uint8_t>{7,0b1111001,10}, + HuffmanCode<uint8_t>{7,0b1111010,11}, + HuffmanCode<uint8_t>{7,0b1111011,12}, + HuffmanCode<uint8_t>{7,0b1111100,13}, + HuffmanCode<uint8_t>{7,0b1111101,14}, + HuffmanCode<uint8_t>{7,0b1111110,15}, + HuffmanCode<uint8_t>{7,0b1111111,0} + }; + auto &lengthDecoder=(_ver==2)?lengthDecoder2:lengthDecoder4; + + uint32_t minBits=1; + + while (!outputStream.eof()) + { + if (!readBits(1)) + { + if (readBit()) + { + outputStream.writeByte(read4Bits(true)); + } else { + if (_ver==4) + { + outputStream.writeByte(literalTable[(read4Bits(false)<<1)+readBit()]); + } else outputStream.writeByte(literalTable[read4Bits(false)]); + } + } else { + uint32_t count=lengthDecoder.decode([&](){return readBits(1);}); + if (!count) + { + count=readBits(4); + if (count==0xfU) + { + count=readBits(8); + if (count==0xffU) throw Decompressor::DecompressionError(); + else count+=3; + } else count+=(_ver==2)?14:16; + } + + uint32_t distance; + bool isMax=false; + do { + uint32_t bits=readBits(3)+minBits; + distance=readBits(bits); + isMax=(distance==((1U<<bits)-1))&&(bits==minBits+7); + if (isMax) minBits++; + distance+=(((1<<bits)-1)&~((1<<minBits)-1))+1; + } while (isMax); + + // buggy compressors + count=std::min(count,uint32_t(rawSize-outputStream.getOffset())); + if (!count) throw Decompressor::DecompressionError(); + + outputStream.copy(distance,count); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LIN2Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LIN2Decompressor.hpp new file mode 100644 index 00000000..4b4878e0 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LIN2Decompressor.hpp @@ -0,0 +1,35 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LIN2DECOMPRESSOR_HPP +#define LIN2DECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class LIN2Decompressor : public XPKDecompressor +{ +public: + LIN2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~LIN2Decompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _ver=0; + size_t _endStreamOffset=0; + size_t _midStreamOffset=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZBSDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZBSDecompressor.cpp new file mode 100644 index 00000000..fc3a826f --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZBSDecompressor.cpp @@ -0,0 +1,75 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LZBSDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool LZBSDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("LZBS"); +} + +std::shared_ptr<XPKDecompressor> LZBSDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<LZBSDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +LZBSDecompressor::LZBSDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr) || _packedData.size()<1) throw Decompressor::InvalidFormatError(); +} + +LZBSDecompressor::~LZBSDecompressor() +{ + // nothing needed +} + +const std::string &LZBSDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-LZBS: LZBS CyberYAFA compressor"; + return name; +} + +void LZBSDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,1,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return rotateBits(bitReader.readBits8(count),count); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + uint32_t bits=0,maxBits=uint32_t(_packedData[0]); + while (!outputStream.eof()) + { + if (!readBits(1)) + { + outputStream.writeByte(readBits(8)); + } else { + uint32_t count=readBits(8)+2; + if (count==2) + { + count=readBits(12); + if (!count) throw Decompressor::DecompressionError(); + for (uint32_t i=0;i<count;i++) + outputStream.writeByte(readBits(8)); + } else { + while (outputStream.getOffset()>=(1ULL<<bits) && bits<maxBits) bits++; + uint32_t distance=readBits(bits); + + outputStream.copy(distance,count); + } + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZBSDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZBSDecompressor.hpp new file mode 100644 index 00000000..57913d6d --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZBSDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LZBSDECOMPRESSOR_HPP +#define LZBSDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class LZBSDecompressor : public XPKDecompressor +{ +public: + LZBSDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~LZBSDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZCBDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZCBDecompressor.cpp new file mode 100644 index 00000000..298ebd29 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZCBDecompressor.cpp @@ -0,0 +1,326 @@ +/* Copyright (C) Teemu Suutari */ + +#include <array> + +#include "LZCBDecompressor.hpp" +#include "RangeDecoder.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +template<size_t T> +class FrequencyTree +{ +public: + FrequencyTree() + { + for (uint32_t i=0;i<_size;i++) + _tree[i]=0; + } + + ~FrequencyTree() + { + // nothing needed + } + + uint16_t decode(uint16_t value,uint16_t &low,uint16_t &freq) const + { + if (value>=_tree[_size-1]) + throw Decompressor::DecompressionError(); + uint16_t symbol=0; + low=0; + for (uint32_t i=_levels-2;;i--) + { + uint16_t tmp=_tree[_levelOffsets[i]+symbol]; + if (uint32_t(symbol+1)<_levelSizes[i] && value>=tmp) + { + symbol++; + low+=tmp; + value-=tmp; + } + if (!i) break; + symbol<<=1; + } + freq=_tree[symbol]; + return symbol; + } + + bool exists(uint16_t symbol) const + { + return _tree[symbol]; + } + + void increment(uint16_t symbol) + { + for (uint16_t i=0;i<_levels;i++) + { + _tree[_levelOffsets[i]+symbol]++; + symbol>>=1; + } + } + + void halve() + { + // non-standard way + for (uint32_t i=0;i<T;i++) + _tree[i]>>=1; + for (uint32_t i=T;i<_size;i++) + _tree[i]=0; + for (uint32_t i=0,length=T;i<_levels-1;i++,length=(length+1)>>1) + { + for (uint32_t j=0;j<length;j++) + _tree[_levelOffsets[i+1]+(j>>1)]+=_tree[_levelOffsets[i]+j]; + } + } + + uint32_t getTotal() const + { + return _tree[_size-1]; + } + +private: + static constexpr uint32_t levelSize(uint32_t level) + { + uint32_t ret=T; + for (uint32_t i=0;i<level;i++) + { + ret=(ret+1)>>1; + } + return ret; + } + + static constexpr uint32_t levels() + { + uint32_t ret=0; + while (levelSize(ret)!=1) ret++; + return ret+1; + } + + static constexpr uint32_t size() + { + uint32_t ret=0; + for (uint32_t i=0;i<levels();i++) + ret+=levelSize(i); + return ret; + } + + static constexpr uint32_t levelOffset(uint32_t level) + { + uint32_t ret=0; + for (uint32_t i=0;i<level;i++) + ret+=levelSize(i); + return ret; + } + + template<uint32_t... I> + static constexpr auto makeLevelOffsetSequence(std::integer_sequence<uint32_t,I...>) + { + return std::integer_sequence<uint32_t,levelOffset(I)...>{}; + } + + template<uint32_t... I> + static constexpr auto makeLevelSizeSequence(std::integer_sequence<uint32_t,I...>) + { + return std::integer_sequence<uint32_t,levelSize(I)...>{}; + } + + template<uint32_t... I> + static constexpr std::array<uint32_t,sizeof...(I)> makeArray(std::integer_sequence<uint32_t,I...>) + { + return std::array<uint32_t,sizeof...(I)>{{I...}}; + } + + static constexpr uint32_t _size=size(); + static constexpr uint32_t _levels=levels(); + static constexpr std::array<uint32_t,_levels> _levelOffsets=makeArray(makeLevelOffsetSequence(std::make_integer_sequence<uint32_t,levels()>{})); + static constexpr std::array<uint32_t,_levels> _levelSizes=makeArray(makeLevelSizeSequence(std::make_integer_sequence<uint32_t,levels()>{})); + + uint16_t _tree[size()]; +}; + +template<size_t T> +class FrequencyDecoder +{ +public: + FrequencyDecoder(RangeDecoder &decoder) : + _decoder(decoder) + { + // nothing needed + } + + ~FrequencyDecoder() + { + // nothing needed + } + + template<typename F> + uint16_t decode(F readFunc) + { + uint16_t freq=0,symbol,value=_decoder.decode(_threshold+_tree.getTotal()); + if (value>=_threshold) + { + uint16_t low; + symbol=_tree.decode(value-_threshold,low,freq); + _decoder.scale(_threshold+low,_threshold+low+freq,_threshold+_tree.getTotal()); + if (freq==1 && _threshold>1) + _threshold--; + } else { + _decoder.scale(0,_threshold,_threshold+_tree.getTotal()); + symbol=readFunc(); + // A bug in the encoder + if (!symbol && _tree.exists(symbol)) symbol=T; + _threshold++; + } + _tree.increment(symbol); + if (_threshold+_tree.getTotal()>=0x3ffdU) + { + _tree.halve(); + _threshold=(_threshold>>1)+1; + } + return symbol; + } + +private: + RangeDecoder &_decoder; + FrequencyTree<T+1> _tree; + uint16_t _threshold=1; +}; + +bool LZCBDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("LZCB"); +} + +std::shared_ptr<XPKDecompressor> LZCBDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<LZCBDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +LZCBDecompressor::LZCBDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (packedData.size()<2) throw Decompressor::InvalidFormatError(); +} + +LZCBDecompressor::~LZCBDecompressor() +{ + // nothing needed +} + +const std::string &LZCBDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-LZCB: LZ-compressor"; + return name; +} + +void LZCBDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + class BitReader : public RangeDecoder::BitReader + { + public: + BitReader(ForwardInputStream &stream) : + _reader(stream) + { + // nothing needed + } + + virtual ~BitReader() + { + // nothing needed + } + + virtual uint32_t readBit() override final + { + return _reader.readBitsBE32(1); + } + + uint32_t readBits(uint32_t bitCount) + { + return _reader.readBitsBE32(bitCount); + } + + private: + MSBBitReader<ForwardInputStream> _reader; + }; + + ForwardInputStream inputStream(_packedData,0,_packedData.size(),true); + ForwardOutputStream outputStream(rawData,0,rawData.size()); + BitReader bitReader(inputStream); + + RangeDecoder rangeDecoder(bitReader,bitReader.readBits(16)); + + // Ugly duplicates + auto readByte=[&]()->uint16_t + { + uint16_t ret=rangeDecoder.decode(0x100U); + rangeDecoder.scale(ret,ret+1,0x100U); + return ret; + }; + + auto readCount=[&]()->uint16_t + { + uint16_t ret=rangeDecoder.decode(0x101U); + rangeDecoder.scale(ret,ret+1,0x101U); + return ret; + }; + + FrequencyDecoder<256> baseLiteralDecoder(rangeDecoder); + FrequencyDecoder<257> repeatCountDecoder(rangeDecoder); + FrequencyDecoder<257> literalCountDecoder(rangeDecoder); + FrequencyDecoder<256> distanceDecoder(rangeDecoder); + + std::unique_ptr<FrequencyDecoder<256>> literalDecoders[256]; + + uint8_t ch=uint8_t(baseLiteralDecoder.decode(readByte)); + outputStream.writeByte(ch); + bool lastIsLiteral=true; + while (!outputStream.eof()) + { + uint32_t count=repeatCountDecoder.decode(readCount); + if (count) + { + if (count==0x100U) + { + uint32_t tmp; + do + { + tmp=readByte(); + count+=tmp; + } while (tmp==0xffU); + } + count+=lastIsLiteral?5:4; + + uint32_t distance=distanceDecoder.decode(readByte)<<8; + distance|=readByte(); + + ch=outputStream.copy(distance,count); + lastIsLiteral=false; + } else { + uint16_t literalCount; + do + { + literalCount=literalCountDecoder.decode(readCount); + if (!literalCount) throw Decompressor::DecompressionError(); + + for (uint32_t i=0;i<literalCount;i++) + { + auto &literalDecoder=literalDecoders[ch]; + if (!literalDecoder) literalDecoder=std::make_unique<FrequencyDecoder<256>>(rangeDecoder); + ch=uint8_t(literalDecoder->decode([&]() + { + return baseLiteralDecoder.decode(readByte); + })); + outputStream.writeByte(ch); + } + } while (literalCount==0x100U); + lastIsLiteral=true; + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZCBDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZCBDecompressor.hpp new file mode 100644 index 00000000..bbd8958f --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZCBDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LZCBDECOMPRESSOR_HPP +#define LZCBDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class LZCBDecompressor : public XPKDecompressor +{ +public: + LZCBDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~LZCBDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW2Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW2Decompressor.cpp new file mode 100644 index 00000000..2d68abf3 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW2Decompressor.cpp @@ -0,0 +1,74 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LZW2Decompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool LZW2Decompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("LZW2") || hdr==FourCC("LZW3"); +} + +std::shared_ptr<XPKDecompressor> LZW2Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<LZW2Decompressor>(hdr,recursionLevel,packedData,state,verify); +} + +LZW2Decompressor::LZW2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); + _ver=(hdr==FourCC("LZW2"))?2:3; +} + +LZW2Decompressor::~LZW2Decompressor() +{ + // nothing needed +} + +const std::string &LZW2Decompressor::getSubName() const noexcept +{ + static std::string name2="XPK-LZW2: LZW2 CyberYAFA compressor"; + static std::string name3="XPK-LZW3: LZW3 CyberYAFA compressor"; + return (_ver==2)?name2:name3; +} + +void LZW2Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + LSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBit=[&]()->uint32_t + { + return bitReader.readBitsBE32(1); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + while (!outputStream.eof()) + { + if (!readBit()) + { + outputStream.writeByte(readByte()); + } else { + uint32_t distance=uint32_t(readByte())<<8; + distance|=uint32_t(readByte()); + if (!distance) throw Decompressor::DecompressionError(); + distance=65536-distance; + uint32_t count=uint32_t(readByte())+4; + + outputStream.copy(distance,count); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW2Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW2Decompressor.hpp new file mode 100644 index 00000000..79d251bc --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW2Decompressor.hpp @@ -0,0 +1,33 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LZW2DECOMPRESSOR_HPP +#define LZW2DECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class LZW2Decompressor : public XPKDecompressor +{ +public: + LZW2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~LZW2Decompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _ver=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW4Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW4Decompressor.cpp new file mode 100644 index 00000000..02c6c443 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW4Decompressor.cpp @@ -0,0 +1,72 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LZW4Decompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool LZW4Decompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("LZW4"); +} + +std::shared_ptr<XPKDecompressor> LZW4Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<LZW4Decompressor>(hdr,recursionLevel,packedData,state,verify); +} + +LZW4Decompressor::LZW4Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +LZW4Decompressor::~LZW4Decompressor() +{ + // nothing needed +} + +const std::string &LZW4Decompressor::getSubName() const noexcept +{ + static std::string name="XPK-LZW4: LZW4 CyberYAFA compressor"; + return name; +} + +void LZW4Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBit=[&]()->uint32_t + { + return bitReader.readBitsBE32(1); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + while (!outputStream.eof()) + { + if (!readBit()) + { + outputStream.writeByte(readByte()); + } else { + uint32_t distance=uint32_t(readByte())<<8; + distance|=uint32_t(readByte()); + if (!distance) throw Decompressor::DecompressionError(); + distance=65536-distance; + uint32_t count=uint32_t(readByte())+3; + + outputStream.copy(distance,count); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW4Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW4Decompressor.hpp new file mode 100644 index 00000000..6bd0abf2 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW4Decompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LZW4DECOMPRESSOR_HPP +#define LZW4DECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class LZW4Decompressor : public XPKDecompressor +{ +public: + LZW4Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~LZW4Decompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW5Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW5Decompressor.cpp new file mode 100644 index 00000000..b32fa10e --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW5Decompressor.cpp @@ -0,0 +1,100 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LZW5Decompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool LZW5Decompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("LZW5"); +} + +std::shared_ptr<XPKDecompressor> LZW5Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<LZW5Decompressor>(hdr,recursionLevel,packedData,state,verify); +} + +LZW5Decompressor::LZW5Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +LZW5Decompressor::~LZW5Decompressor() +{ + // nothing needed +} + +const std::string &LZW5Decompressor::getSubName() const noexcept +{ + static std::string name="XPK-LZW5: LZW5 CyberYAFA compressor"; + return name; +} + +void LZW5Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto read2Bits=[&]()->uint32_t + { + return bitReader.readBitsBE32(2); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + while (!outputStream.eof()) + { + uint32_t distance,count; + + auto readld=[&]()->uint32_t + { + uint32_t ret=uint32_t(readByte())<<8; + ret|=uint32_t(readByte()); + if (!ret) throw Decompressor::DecompressionError(); + return ret; + }; + + switch (read2Bits()) + { + case 0: + outputStream.writeByte(readByte()); + break; + + case 1: + distance=readld(); + count=(distance&3)+2; + distance=0x4000-(distance>>2); + outputStream.copy(distance,count); + break; + + case 2: + distance=readld(); + count=(distance&15)+2; + distance=0x1000-(distance>>4); + outputStream.copy(distance,count); + break; + + case 3: + distance=readld(); + count=uint32_t(readByte())+3; + distance=0x10000-distance; + outputStream.copy(distance,count); + break; + + default: + throw Decompressor::DecompressionError(); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW5Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW5Decompressor.hpp new file mode 100644 index 00000000..8ce65b41 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZW5Decompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LZW5DECOMPRESSOR_HPP +#define LZW5DECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class LZW5Decompressor : public XPKDecompressor +{ +public: + LZW5Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~LZW5Decompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZXDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZXDecompressor.cpp new file mode 100644 index 00000000..cdd29cb7 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZXDecompressor.cpp @@ -0,0 +1,245 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstdint> +#include <cstring> + +#include "LZXDecompressor.hpp" +#include "HuffmanDecoder.hpp" +#include "DLTADecode.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/CRC32.hpp" +#include "common/Common.hpp" +#include "common/OverflowCheck.hpp" + + +namespace ancient::internal +{ + +bool LZXDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("ELZX") || hdr==FourCC("SLZX"); +} + +std::shared_ptr<XPKDecompressor> LZXDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<LZXDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +LZXDecompressor::LZXDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); + if (hdr==FourCC("SLZX")) _isSampled=true; + // There is no good spec on the LZX header content -> lots of unknowns here + if (_packedData.size()<41) throw Decompressor::InvalidFormatError(); + // XPK LZX compression is embedded single file of LZX -> read first file. Ignore rest + // this will include flags, which need to be zero anyway + uint32_t streamHdr=_packedData.readBE32(0); + if (streamHdr!=FourCC("LZX\0")) throw Decompressor::InvalidFormatError(); + + _rawSize=_packedData.readLE32(12); + _packedSize=_packedData.readLE32(16); + + _rawCRC=_packedData.readLE32(32); + uint32_t headerCRC=_packedData.readLE32(36); + + uint8_t tmp=_packedData.read8(21); + if (tmp && tmp!=2) throw Decompressor::InvalidFormatError(); + if (tmp==2) _isCompressed=true; + + _packedOffset=41U+_packedData.read8(40U); + _packedOffset+=_packedData.read8(24U); + _packedSize+=_packedOffset; + + if (_packedSize>_packedData.size()) throw Decompressor::InvalidFormatError(); + if (verify) + { + uint32_t crc=CRC32(_packedData,10,26,0); + for (uint32_t i=0;i<4;i++) crc=CRC32Byte(0,crc); + crc=CRC32(_packedData,40,_packedOffset-40,crc); + if (crc!=headerCRC) throw Decompressor::VerificationError(); + } +} + +LZXDecompressor::~LZXDecompressor() +{ + // nothing needed +} + +const std::string &LZXDecompressor::getSubName() const noexcept +{ + static std::string nameE="XPK-ELZX: LZX-compressor"; + static std::string nameS="XPK-SLZX: LZX-compressor with delta encoding"; + return (_isSampled)?nameS:nameE; +} + +void LZXDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); + if (!_isCompressed) + { + if (_packedSize!=_rawSize) throw Decompressor::DecompressionError(); + std::memcpy(rawData.data(),_packedData.data()+_packedOffset,_rawSize); + return; + } + + ForwardInputStream inputStream(_packedData,_packedOffset,_packedSize); + LSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBitsBE16(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBitsBE16(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + typedef HuffmanDecoder<uint32_t> LZXDecoder; + + // possibly padded/reused later if multiple blocks + uint8_t literalTable[768]; + for (uint32_t i=0;i<768;i++) literalTable[i]=0; + LZXDecoder literalDecoder; + uint32_t previousDistance=1; + + while (!outputStream.eof()) + { + + auto createHuffmanTable=[&](LZXDecoder &dec,const uint8_t *bitLengths,uint32_t bitTableLength) + { + uint8_t minDepth=16,maxDepth=0; + for (uint32_t i=0;i<bitTableLength;i++) + { + if (bitLengths[i] && bitLengths[i]<minDepth) minDepth=bitLengths[i]; + if (bitLengths[i]>maxDepth) maxDepth=bitLengths[i]; + } + if (!maxDepth) return; + + dec.createOrderlyHuffmanTable(bitLengths,bitTableLength); + }; + + uint32_t method=readBits(3); + if (method<1 || method>3) throw Decompressor::DecompressionError(); + + LZXDecoder distanceDecoder; + if (method==3) + { + uint8_t bitLengths[8]; + for (uint32_t i=0;i<8;i++) bitLengths[i]=readBits(3); + createHuffmanTable(distanceDecoder,bitLengths,8); + } + + size_t blockLength=readBits(8)<<16; + blockLength|=readBits(8)<<8; + blockLength|=readBits(8); + if (OverflowCheck::sum(blockLength,outputStream.getOffset())>_rawSize) throw Decompressor::DecompressionError(); + + if (method!=1) + { + literalDecoder.reset(); + for (uint32_t pos=0,block=0;block<2;block++) + { + uint32_t adjust=(block)?0:1; + uint32_t maxPos=(block)?768:256; + LZXDecoder bitLengthDecoder; + { + uint8_t lengthTable[20]; + for (uint32_t i=0;i<20;i++) lengthTable[i]=readBits(4); + createHuffmanTable(bitLengthDecoder,lengthTable,20); + } + while (pos<maxPos) + { + uint32_t symbol=bitLengthDecoder.decode(readBit); + + auto doRepeat=[&](uint32_t count,uint8_t value) + { + if (count>maxPos-pos) count=maxPos-pos; + while (count--) literalTable[pos++]=value; + }; + + auto symDecode=[&](uint32_t value)->uint32_t + { + return (literalTable[pos]+17-value)%17; + }; + + switch (symbol) + { + case 17: + doRepeat(readBits(4)+3+adjust,0); + break; + + case 18: + doRepeat(readBits(6-adjust)+19+adjust,0); + break; + + case 19: + { + uint32_t count=readBit()+3+adjust; + doRepeat(count,symDecode(bitLengthDecoder.decode(readBit))); + } + break; + + default: + literalTable[pos++]=symDecode(symbol); + break; + } + } + } + createHuffmanTable(literalDecoder,literalTable,768); + } + + while (blockLength) + { + uint32_t symbol=literalDecoder.decode(readBit); + if (symbol<256) { + outputStream.writeByte(symbol); + blockLength--; + } else { + // both of these tables are almost too regular to be tables... + static const uint8_t ldBits[32]={ + 0,0,0,0,1,1,2,2, + 3,3,4,4,5,5,6,6, + 7,7,8,8,9,9,10,10, + 11,11,12,12,13,13,14,14}; + static const uint32_t ldAdditions[32]={ + 0x0, + 0x1, 0x2, 0x3, 0x4, 0x6, 0x8, 0xc, 0x10, + 0x18, 0x20, 0x30, 0x40, 0x60, 0x80, 0xc0, 0x100, + 0x180, 0x200, 0x300, 0x400, 0x600, 0x800, 0xc00,0x1000, + 0x1800,0x2000,0x3000,0x4000,0x6000,0x8000,0xc000}; + + symbol-=256; + uint32_t bits=ldBits[symbol&0x1f]; + uint32_t distance=ldAdditions[symbol&0x1f]; + if (bits>=3 && method==3) + { + distance+=readBits(bits-3)<<3; + uint32_t tmp=distanceDecoder.decode(readBit); + distance+=tmp; + } else { + distance+=readBits(bits); + if (!distance) distance=previousDistance; + } + previousDistance=distance; + + uint32_t count=ldAdditions[symbol>>5]+readBits(ldBits[symbol>>5])+3; + if (count>blockLength) throw Decompressor::DecompressionError(); + outputStream.copy(distance,count); + blockLength-=count; + } + } + } + if (verify) + { + uint32_t crc=CRC32(rawData,0,_rawSize,0); + if (crc!=_rawCRC) throw Decompressor::VerificationError(); + } + if (_isSampled) + DLTADecode::decode(rawData,rawData,0,_rawSize); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZXDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZXDecompressor.hpp new file mode 100644 index 00000000..396b8220 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/LZXDecompressor.hpp @@ -0,0 +1,37 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LZXDECOMPRESSOR_HPP +#define LZXDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class LZXDecompressor : public XPKDecompressor +{ +public: + LZXDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + virtual ~LZXDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + bool _isSampled=false; + bool _isCompressed=false; + size_t _packedSize=0; + size_t _packedOffset=0; + size_t _rawSize=0; + uint32_t _rawCRC=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH1Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH1Decompressor.cpp new file mode 100644 index 00000000..db1917e4 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH1Decompressor.cpp @@ -0,0 +1,90 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LH1Decompressor.hpp" + +#include "../HuffmanDecoder.hpp" +#include "../DynamicHuffmanDecoder.hpp" +#include "../InputStream.hpp" +#include "../OutputStream.hpp" + +namespace ancient::internal +{ + +LH1Decompressor::LH1Decompressor(const Buffer &packedData) : + _packedData(packedData) +{ + // nothing needed +} + +LH1Decompressor::~LH1Decompressor() +{ + // nothing needed +} + +size_t LH1Decompressor::getRawSize() const noexcept +{ + // N/A + return 0; +} + +size_t LH1Decompressor::getPackedSize() const noexcept +{ + // N/A + return 0; +} + +const std::string &LH1Decompressor::getName() const noexcept +{ + static std::string name="LHA: LH1"; + return name; +} + +void LH1Decompressor::decompressImpl(Buffer &rawData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + DynamicHuffmanDecoder<314> decoder; + + static const uint8_t distanceHighBits[64]={ + 3,4,4,4,5,5,5,5, 5,5,5,5,6,6,6,6, + 6,6,6,6,6,6,6,6, 7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7, + 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8 + }; + + HuffmanDecoder<uint8_t> distanceDecoder; + distanceDecoder.createOrderlyHuffmanTable(distanceHighBits,64); + + while (!outputStream.eof()) + { + uint32_t code=decoder.decode(readBit); + if (decoder.getMaxFrequency()==0x8000U) decoder.halve(); + decoder.update(code); + + if (code<256) + { + outputStream.writeByte(code); + } else { + uint32_t distance=distanceDecoder.decode(readBit); + distance=(distance<<6)|readBits(6); + distance++; + + uint32_t count=code-253; + + outputStream.copy(distance,count,0x20); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH1Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH1Decompressor.hpp new file mode 100644 index 00000000..6b098538 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH1Decompressor.hpp @@ -0,0 +1,30 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LH1DECOMPRESSOR_HPP +#define LH1DECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class LH1Decompressor : public Decompressor +{ +public: + LH1Decompressor(const Buffer &packedData); + virtual ~LH1Decompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH2Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH2Decompressor.cpp new file mode 100644 index 00000000..d9cf96da --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH2Decompressor.cpp @@ -0,0 +1,126 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LH2Decompressor.hpp" + +#include "../HuffmanDecoder.hpp" +#include "../DynamicHuffmanDecoder.hpp" +#include "../InputStream.hpp" +#include "../OutputStream.hpp" + + +namespace ancient::internal +{ + +LH2Decompressor::LH2Decompressor(const Buffer &packedData) : + _packedData(packedData) +{ + // nothing needed +} + +LH2Decompressor::~LH2Decompressor() +{ + // nothing needed +} + +size_t LH2Decompressor::getRawSize() const noexcept +{ + // N/A + return 0; +} + +size_t LH2Decompressor::getPackedSize() const noexcept +{ + // N/A + return 0; +} + +const std::string &LH2Decompressor::getName() const noexcept +{ + static std::string name="LHA: LH2"; + return name; +} + +// This is probably fishiest of the fishy formats I've worked with +// Basically it uses Dynamic Huffman decoder but instead of static +// size, it is dynamically growing. However, that part is not well +// defined. Thus +// a. It is very hard to use LH2 using generic implementation +// instead of the specific one used by LHA +// b. There are bugs in encoder which need to be baked in the decoder +// as well (Probably there is lots of unneccesary stuff in addCode, +// but better safe than sorry) +// c. LH 1.9x and UNLHA32 refuse to use LH2 beyond 8k files. Thus +// we can only guess if the wraparound is correct, since nothing +// should use it +// jLHA in theory supports LH2 without limitations, but it produces +// broken bitstream. +// +// So far this works with the files I've tried from "official" LHA +void LH2Decompressor::decompressImpl(Buffer &rawData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + DynamicHuffmanDecoder<286> valueDecoder; + DynamicHuffmanDecoder<128> distanceDecoder(0); + uint32_t distancePosition=0; + + uint32_t distCount=0; + while (!outputStream.eof()) + { + uint32_t code=valueDecoder.decode(readBit); + if (valueDecoder.getMaxFrequency()==0x8000U) valueDecoder.halve(); + valueDecoder.update(code); + + if (code<256) + { + outputStream.writeByte(code); + } else { + if (code==285) code+=readBits(8); + uint32_t count=code-253; + + // Bug in LH2 where count does not match frequency. + // Makes things more complicated + auto updateDist=[&](uint32_t code) + { + if (distCount==0x8000U) + { + distanceDecoder.halve(); + distCount=distanceDecoder.getMaxFrequency(); + } + distanceDecoder.update(code); + distCount++; + }; + + uint32_t maxDist=std::min(uint32_t((outputStream.getOffset()+63)>>6),128U); + if (distancePosition!=maxDist) + { + for (uint32_t i=distancePosition;i<maxDist;i++) + { + distanceDecoder.addCode(); + updateDist(i); + } + distancePosition=maxDist; + } + + uint32_t distance=distanceDecoder.decode(readBit); + updateDist(distance); + distance=(distance<<6)|readBits(6); + distance++; + + outputStream.copy(distance,count,0x20); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH2Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH2Decompressor.hpp new file mode 100644 index 00000000..d406585d --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH2Decompressor.hpp @@ -0,0 +1,30 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LH2DECOMPRESSOR_HPP +#define LH2DECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class LH2Decompressor : public Decompressor +{ +public: + LH2Decompressor(const Buffer &packedData); + virtual ~LH2Decompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH3Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH3Decompressor.cpp new file mode 100644 index 00000000..f9a62357 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH3Decompressor.cpp @@ -0,0 +1,149 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LH3Decompressor.hpp" + +#include "../HuffmanDecoder.hpp" +#include "../InputStream.hpp" +#include "../OutputStream.hpp" + + +namespace ancient::internal +{ + +LH3Decompressor::LH3Decompressor(const Buffer &packedData) : + _packedData(packedData) +{ + // nothing needed +} + +LH3Decompressor::~LH3Decompressor() +{ + // nothing needed +} + +size_t LH3Decompressor::getRawSize() const noexcept +{ + // N/A + return 0; +} + +size_t LH3Decompressor::getPackedSize() const noexcept +{ + // N/A + return 0; +} + +const std::string &LH3Decompressor::getName() const noexcept +{ + static std::string name="LHA: LH3"; + return name; +} + +void LH3Decompressor::decompressImpl(Buffer &rawData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + OptionalHuffmanDecoder<uint32_t> decoder; + + static const uint8_t distanceHighBits[128]={ + 2,4,4,5,5,5,6,6, 6,6,6,6,6,7,7,7, + 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,8, + 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, + + 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,9,9, + 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9 + }; + + OptionalHuffmanDecoder<uint8_t> distanceDecoder; + + uint32_t blockRemaining=0; + while (!outputStream.eof()) + { + if (!blockRemaining) + { + blockRemaining=readBits(16); + if (!blockRemaining) blockRemaining=0x10000; + + // not strictly needed as a lambda, but cleaner this way + auto createDecoderTable=[&]() + { + decoder.reset(); + + uint8_t symbolBits[286]; + uint32_t oneCount=0; + for (uint32_t i=0;i<286;i++) + { + if (readBit()) symbolBits[i]=readBits(4)+1; + else symbolBits[i]=0; + if (symbolBits[i]==1) oneCount++; + if (i==2 && oneCount==3) + { + decoder.setEmpty(readBits(9)); + return; + } + } + decoder.createOrderlyHuffmanTable(symbolBits,286); + }; + + // ditto + auto createDistanceDecoderTable=[&]() + { + distanceDecoder.reset(); + + if (readBit()) + { + uint8_t symbolBits[128]; + uint32_t oneCount=0; + for (uint32_t i=0;i<128;i++) + { + symbolBits[i]=readBits(4); + if (symbolBits[i]==1) oneCount++; + if (i==2 && oneCount==3) + { + // 7 bits would be fine, but whatever + distanceDecoder.setEmpty(readBits(9)); + return; + } + } + distanceDecoder.createOrderlyHuffmanTable(symbolBits,128); + } else distanceDecoder.createOrderlyHuffmanTable(distanceHighBits,128); + }; + + createDecoderTable(); + createDistanceDecoderTable(); + } + blockRemaining--; + + uint32_t code=decoder.decode(readBit); + + if (code<256) + { + outputStream.writeByte(code); + } else { + if (code==285) code+=readBits(8); + uint32_t distance=distanceDecoder.decode(readBit); + distance=(distance<<6)|readBits(6); + distance++; + + uint32_t count=code-253; + + outputStream.copy(distance,count,0x20); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH3Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH3Decompressor.hpp new file mode 100644 index 00000000..8676c1fe --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LH3Decompressor.hpp @@ -0,0 +1,30 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LH3DECOMPRESSOR_HPP +#define LH3DECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class LH3Decompressor : public Decompressor +{ +public: + LH3Decompressor(const Buffer &packedData); + virtual ~LH3Decompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LHXDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LHXDecompressor.cpp new file mode 100644 index 00000000..144005e9 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LHXDecompressor.cpp @@ -0,0 +1,191 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LHXDecompressor.hpp" + +#include "../HuffmanDecoder.hpp" +#include "../InputStream.hpp" +#include "../OutputStream.hpp" + + +namespace ancient::internal +{ + +LHXDecompressor::LHXDecompressor(const Buffer &packedData) : + _packedData(packedData), + _method(0) +{ + // nothing needed +} + +LHXDecompressor::LHXDecompressor(const Buffer &packedData,uint32_t method) : + _packedData(packedData), + _method(method-3) +{ + if (method<4 || method>8) throw InvalidFormatError(); +} + +LHXDecompressor::~LHXDecompressor() +{ + // nothing needed +} + +size_t LHXDecompressor::getRawSize() const noexcept +{ + // N/A + return 0; +} + +size_t LHXDecompressor::getPackedSize() const noexcept +{ + // N/A + return 0; +} + +const std::string &LHXDecompressor::getName() const noexcept +{ + static std::string name="LHA: LH4, LH5, LH6, LH7, LH8, LHX"; + return name; +} + +void LHXDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + OptionalHuffmanDecoder<uint32_t> decoder; + OptionalHuffmanDecoder<uint32_t> distanceDecoder; + + static const struct { + uint32_t mask; + uint32_t distanceTableSize; + uint32_t distanceBits; + } methodTable[6] = { + {0x7ffffU,20,5}, // LHX + {0xfffU,14,4}, // LH4 + {0x1fffU,14,4}, // LH5 + {0x7fffU,16,5}, // LH6 + {0xffffU,17,5}, // LH7 + {0xffffU,17,5} // LH8 + // LH8 is the only Joe Jared method seen. + // It exists in early versions of LH7-tool + // In those versions LH8 is just synonym for LH7 + // However, LH9+ is something I have not seen at + // all (i.e. not in DLH7021Q/WLH7021Q.) + // There would be one more version (0.21R) but + // that seems lost. Most likely minor bump from Q->R + // do not enable those formats either and the LH9+ + // formats are nothing but hopes for "future work" + // that never materialized (until I'm proven wrong ofc) + }; + + uint32_t blockRemaining=0; + while (!outputStream.eof()) + { + if (!blockRemaining) + { + blockRemaining=readBits(16); + if (!blockRemaining) blockRemaining=0x10000; + + auto createTable=[&](OptionalHuffmanDecoder<uint32_t> &dest,uint32_t count,uint32_t bits,bool enableHole) + { + uint8_t symbolBits[20]; + uint32_t length=readBits(bits); + if (!length) + { + dest.setEmpty(readBits(bits)); + } else if (length<=count) { + for (uint32_t i=0;i<length;) + { + uint32_t value=readBits(3); + if (value==7) + while (readBit()) value++; + if (value>32) throw DecompressionError(); + symbolBits[i++]=value; + if (i==3 && enableHole) + { + uint32_t zeros=readBits(2); + if (i+zeros>length) throw DecompressionError(); + for (uint32_t j=0;j<zeros;j++) symbolBits[i++]=0; + } + } + dest.createOrderlyHuffmanTable(symbolBits,length); + } else throw DecompressionError(); + }; + + OptionalHuffmanDecoder<uint32_t> tmpDecoder; + createTable(tmpDecoder,19,5,true); + + decoder.reset(); + + uint8_t symbolBits[511]; + uint32_t length=readBits(9); + if (!length) + { + decoder.setEmpty(readBits(9)); + } else { + for (uint32_t i=0;i<length;) + { + uint32_t value=tmpDecoder.decode(readBit); + uint32_t rep; + switch (value) + { + case 0: + value=0; + rep=1; + break; + + case 1: + value=0; + rep=readBits(4)+3; + break; + + case 2: + value=0; + rep=readBits(9)+20; + break; + + default: + value-=2; + rep=1; + break; + } + if (i+rep>length) throw DecompressionError(); + for (uint32_t j=0;j<rep;j++) symbolBits[i++]=value; + } + decoder.createOrderlyHuffmanTable(symbolBits,length); + } + + distanceDecoder.reset(); + createTable(distanceDecoder,methodTable[_method].distanceTableSize,methodTable[_method].distanceBits,false); + } + blockRemaining--; + + uint32_t code=decoder.decode(readBit); + + if (code<256) + { + outputStream.writeByte(code); + } else { + uint32_t distanceBits=distanceDecoder.decode(readBit); + uint32_t distance=distanceBits?((1<<(distanceBits-1))|readBits(distanceBits-1)):0; + distance&=methodTable[_method].mask; // this is theoretical, but original LH has it + distance++; + + uint32_t count=code-253; + + outputStream.copy(distance,count,0x20); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LHXDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LHXDecompressor.hpp new file mode 100644 index 00000000..2cb88346 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LHXDecompressor.hpp @@ -0,0 +1,34 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LHXDECOMPRESSOR_HPP +#define LHXDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class LHXDecompressor : public Decompressor +{ +public: + // by default LHX + LHXDecompressor(const Buffer &packedDatam); + LHXDecompressor(const Buffer &packedDatam,uint32_t method); + virtual ~LHXDecompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + +private: + const Buffer &_packedData; + + uint32_t _method; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZ5Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZ5Decompressor.cpp new file mode 100644 index 00000000..9d7b9ee0 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZ5Decompressor.cpp @@ -0,0 +1,86 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LZ5Decompressor.hpp" + +#include "common/StaticBuffer.hpp" + +#include "../InputStream.hpp" +#include "../OutputStream.hpp" + + +namespace ancient::internal +{ + +LZ5Decompressor::LZ5Decompressor(const Buffer &packedData) : + _packedData(packedData) +{ + // nothing needed +} + +LZ5Decompressor::~LZ5Decompressor() +{ + // nothing needed +} + +size_t LZ5Decompressor::getRawSize() const noexcept +{ + // N/A + return 0; +} + +size_t LZ5Decompressor::getPackedSize() const noexcept +{ + // N/A + return 0; +} + +const std::string &LZ5Decompressor::getName() const noexcept +{ + static std::string name="LHA: LZ5"; + return name; +} + +void LZ5Decompressor::decompressImpl(Buffer &rawData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + LSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + auto readByte=[&]()->uint32_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + StaticBuffer<4096> prevBuffer; + { + uint8_t *bufPtr=prevBuffer.data(); + + for (uint32_t i=0;i<18;i++) *(bufPtr++)=0; + for (uint32_t i=0;i<256;i++) + for (uint32_t j=0;j<13;j++) *(bufPtr++)=i; + for (uint32_t i=0;i<256;i++) *(bufPtr++)=i; + for (uint32_t i=0;i<256;i++) *(bufPtr++)=255-i; + for (uint32_t i=0;i<128;i++) *(bufPtr++)=0; + for (uint32_t i=0;i<110;i++) *(bufPtr++)=' '; + } + + while (!outputStream.eof()) + { + if (readBit()) + { + outputStream.writeByte(readByte()); + } else { + uint32_t byte1=readByte(); + uint32_t byte2=readByte(); + uint32_t distance=((outputStream.getOffset()-byte1-((byte2&0xf0U)<<4)-19)&0xfffU)+1; + uint32_t count=(byte2&0xfU)+3; + + outputStream.copy(distance,count,prevBuffer); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZ5Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZ5Decompressor.hpp new file mode 100644 index 00000000..7141a36f --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZ5Decompressor.hpp @@ -0,0 +1,30 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LZ5DECOMPRESSOR_HPP +#define LZ5DECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class LZ5Decompressor : public Decompressor +{ +public: + LZ5Decompressor(const Buffer &packedData); + virtual ~LZ5Decompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZHDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZHDecompressor.cpp new file mode 100644 index 00000000..9d7a9be3 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZHDecompressor.cpp @@ -0,0 +1,167 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstdint> +#include <cstring> + +#include <map> + +#include "LZHDecompressor.hpp" + +#include "LH1Decompressor.hpp" +#include "LH2Decompressor.hpp" +#include "LH3Decompressor.hpp" +#include "LHXDecompressor.hpp" +#include "LZ5Decompressor.hpp" +#include "LZSDecompressor.hpp" +#include "PMDecompressor.hpp" + + +namespace ancient::internal +{ + +LZHDecompressor::LZHDecompressor(const Buffer &packedData,const std::string &method) : + _packedData(packedData), + _method(method) +{ + // nothing needed +} + +LZHDecompressor::~LZHDecompressor() +{ + // nothing needed +} + +size_t LZHDecompressor::getRawSize() const noexcept +{ + // N/A + return 0; +} + +size_t LZHDecompressor::getPackedSize() const noexcept +{ + // N/A + return 0; +} + +const std::string &LZHDecompressor::getName() const noexcept +{ + static std::string name="Lzh"; + return name; +} + +void LZHDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + enum class Compressor + { + LH0=0, + LH1, + LH2, + LH3, + LH4, + LH5, + LH6, + LH7, + LH8, + LHX, + LZ4, + LZ5, + LZS, + PM0, + PM1, + PM2 + }; + + static std::map<std::string,Compressor> compressorMap{ + {"-lh0-",Compressor::LH0}, + {"-lh1-",Compressor::LH1}, + {"-lh2-",Compressor::LH2}, + {"-lh3-",Compressor::LH3}, + {"-lh4-",Compressor::LH4}, + {"-lh5-",Compressor::LH5}, + {"-lh6-",Compressor::LH6}, + {"-lh7-",Compressor::LH7}, + {"-lh8-",Compressor::LH8}, + {"-lhx-",Compressor::LHX}, + {"-lz4-",Compressor::LZ4}, + {"-lz5-",Compressor::LZ5}, + {"-lzs-",Compressor::LZS}, + {"-pm0-",Compressor::PM0}, + {"-pm1-",Compressor::PM1}, + {"-pm2-",Compressor::PM2} + }; + + auto it=compressorMap.find(_method); + if (it==compressorMap.end()) throw DecompressionError(); + switch (it->second) + { + case Compressor::LH0: + case Compressor::LZ4: + case Compressor::PM0: + if (rawData.size()!=_packedData.size()) throw DecompressionError(); + std::memcpy(rawData.data(),_packedData.data(),rawData.size()); + break; + + case Compressor::LH1: + { + LH1Decompressor dec(_packedData); + dec.decompress(rawData,verify); + } + break; + + case Compressor::LH2: + { + LH2Decompressor dec(_packedData); + dec.decompress(rawData,verify); + } + break; + + case Compressor::LH3: + { + LH3Decompressor dec(_packedData); + dec.decompress(rawData,verify); + } + break; + + case Compressor::LH4: + case Compressor::LH5: + case Compressor::LH6: + case Compressor::LH7: + case Compressor::LH8: + { + LHXDecompressor dec(_packedData,static_cast<uint32_t>(it->second)-static_cast<uint32_t>(Compressor::LH4)+4); + dec.decompress(rawData,verify); + } + break; + + case Compressor::LHX: + { + LHXDecompressor dec(_packedData); + dec.decompress(rawData,verify); + } + break; + + case Compressor::LZ5: + { + LZ5Decompressor dec(_packedData); + dec.decompress(rawData,verify); + } + break; + + case Compressor::LZS: + { + LZSDecompressor dec(_packedData); + dec.decompress(rawData,verify); + } + break; + + case Compressor::PM1: + case Compressor::PM2: + { + PMDecompressor dec(_packedData,static_cast<uint32_t>(it->second)-static_cast<uint32_t>(Compressor::PM1)+1); + dec.decompress(rawData,verify); + } + break; + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZHDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZHDecompressor.hpp new file mode 100644 index 00000000..130487af --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZHDecompressor.hpp @@ -0,0 +1,32 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LZHDECOMPRESSOR_HPP +#define LZHDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class LZHDecompressor : public Decompressor +{ +public: + LZHDecompressor(const Buffer &packedData,const std::string &method); + virtual ~LZHDecompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + +private: + const Buffer &_packedData; + + std::string _method; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZSDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZSDecompressor.cpp new file mode 100644 index 00000000..3f48d486 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZSDecompressor.cpp @@ -0,0 +1,70 @@ +/* Copyright (C) Teemu Suutari */ + +#include "LZSDecompressor.hpp" + +#include "../InputStream.hpp" +#include "../OutputStream.hpp" + + +namespace ancient::internal +{ + +LZSDecompressor::LZSDecompressor(const Buffer &packedData) : + _packedData(packedData) +{ + // nothing needed +} + +LZSDecompressor::~LZSDecompressor() +{ + // nothing needed +} + +size_t LZSDecompressor::getRawSize() const noexcept +{ + // N/A + return 0; +} + +size_t LZSDecompressor::getPackedSize() const noexcept +{ + // N/A + return 0; +} + +const std::string &LZSDecompressor::getName() const noexcept +{ + static std::string name="LHA: LZS"; + return name; +} + +void LZSDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + while (!outputStream.eof()) + { + if (readBit()) + { + outputStream.writeByte(readBits(8)); + } else { + uint32_t distance=((outputStream.getOffset()-readBits(11)-18)&0x7ffU)+1; + uint32_t count=readBits(4)+2; + + outputStream.copy(distance,count,0x20); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZSDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZSDecompressor.hpp new file mode 100644 index 00000000..f0a0c9a7 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/LZSDecompressor.hpp @@ -0,0 +1,30 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef LZSDECOMPRESSOR_HPP +#define LZSDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class LZSDecompressor : public Decompressor +{ +public: + LZSDecompressor(const Buffer &packedData); + virtual ~LZSDecompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/PMDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/PMDecompressor.cpp new file mode 100644 index 00000000..60d9013b --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/PMDecompressor.cpp @@ -0,0 +1,424 @@ +/* Copyright (C) Teemu Suutari */ + +#include "PMDecompressor.hpp" + +#include "../HuffmanDecoder.hpp" +#include "../InputStream.hpp" +#include "../OutputStream.hpp" + + +namespace ancient::internal +{ + +PMDecompressor::PMDecompressor(const Buffer &packedData,uint32_t version) : + _packedData(packedData), + _version(version) +{ + if (version!=1 && version!=2) throw InvalidFormatError(); +} + +PMDecompressor::~PMDecompressor() +{ + // nothing needed +} + +size_t PMDecompressor::getRawSize() const noexcept +{ + // N/A + return 0; +} + +size_t PMDecompressor::getPackedSize() const noexcept +{ + // N/A + return 0; +} + +const std::string &PMDecompressor::getName() const noexcept +{ + static std::string name="LHA: PM1, PM2"; + return name; +} + +uint8_t PMDecompressor::decodeMTF(uint8_t value,uint8_t map[]) +{ + return map[value]; +} + +void PMDecompressor::updateMTF(uint8_t value,uint8_t map[]) +{ + for (uint32_t i=0;;i++) + { + if (map[i]==value) + { + value=i; + break; + } + } + if (value) + { + uint8_t tmp=map[value]; + for (uint32_t i=value;i;i--) + map[i]=map[i-1]; + map[0]=tmp; + } +} + +void PMDecompressor::createMTFMap(uint8_t map[]) +{ + for (uint32_t i=0,j=0x20;j<0x80;i++,j++) map[i]=j; + for (uint32_t i=0x60,j=0;j<0x20;i++,j++) map[i]=j; + for (uint32_t i=0x80,j=0xa0;j<0xe0;i++,j++) map[i]=j; + for (uint32_t i=0xc0,j=0x80;j<0xa0;i++,j++) map[i]=j; + for (uint32_t i=0xe0,j=0xe0;j<0x100;i++,j++) map[i]=j; +} + +void PMDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + if (_version==1) decompressImplPM1(rawData,verify); + else decompressImplPM2(rawData,verify); +} + +void PMDecompressor::decompressImplPM1(Buffer &rawData,bool verify) +{ + static const struct { + uint8_t length; + uint8_t code; + } treeDefinitions[32][6] { + // This is madness, I had to write a special program to decode this. + // Also, in the original there is a bug @ index 17, as identified by + // lhasa (we fix it when creating the decoder) + {{4, 0b0000},{4, 0b0001},{3, 0b001},{2, 0b01},{2, 0b10},{2, 0b11}}, + {{3, 0b000},{3, 0b001},{3, 0b010},{2, 0b10},{2, 0b11},{3, 0b011}}, + {{3, 0b000},{3, 0b001},{2, 0b01},{2, 0b10},{3, 0b110},{3, 0b111}}, + {{2, 0b00},{3, 0b010},{3, 0b011},{2, 0b10},{3, 0b110},{3, 0b111}}, + {{2, 0b00},{3, 0b010},{2, 0b10},{3, 0b011},{3, 0b110},{3, 0b111}}, + {{2, 0b00},{3, 0b010},{2, 0b10},{2, 0b11},{4, 0b0110},{4, 0b0111}}, + {{2, 0b00},{2, 0b01},{3, 0b100},{3, 0b101},{3, 0b110},{3, 0b111}}, + {{2, 0b00},{2, 0b01},{3, 0b100},{2, 0b11},{4, 0b1010},{4, 0b1011}}, + + {{2, 0b00},{2, 0b01},{2, 0b10},{3, 0b110},{4, 0b1110},{4, 0b1111}}, + {{1, 0b0},{4, 0b1000},{3, 0b101},{3, 0b110},{3, 0b111},{4, 0b1001}}, + {{1, 0b0},{4, 0b1000},{3, 0b101},{2, 0b11},{5,0b10010},{5,0b10011}}, + {{1, 0b0},{4, 0b1000},{4, 0b1001},{3, 0b101},{3, 0b110},{3, 0b111}}, + {{1, 0b0},{3, 0b100},{4, 0b1010},{3, 0b110},{3, 0b111},{4, 0b1011}}, + {{1, 0b0},{3, 0b100},{3, 0b101},{3, 0b110},{4, 0b1110},{4, 0b1111}}, + {{1, 0b0},{3, 0b100},{2, 0b11},{4, 0b1010},{5,0b10110},{5,0b10111}}, + {{1, 0b0},{2, 0b10},{4, 0b1100},{4, 0b1101},{4, 0b1110},{4, 0b1111}}, + + {{1, 0b0},{2, 0b10},{3, 0b110},{4, 0b1110},{5,0b11110},{5,0b11111}}, + {{3, 0b000},{3, 0b001},{2, 0b01},{2, 0b10},{2, 0b11},{0, 0b0}}, + {{2, 0b00},{3, 0b010},{2, 0b10},{2, 0b11},{3, 0b011},{0, 0b0}}, + {{2, 0b00},{2, 0b01},{2, 0b10},{3, 0b110},{3, 0b111},{0, 0b0}}, + {{1, 0b0},{4, 0b1000},{3, 0b101},{2, 0b11},{4, 0b1001},{0, 0b0}}, + {{1, 0b0},{3, 0b100},{3, 0b101},{3, 0b110},{3, 0b111},{0, 0b0}}, + {{1, 0b0},{3, 0b100},{2, 0b11},{4, 0b1010},{4, 0b1011},{0, 0b0}}, + {{1, 0b0},{2, 0b10},{3, 0b110},{4, 0b1110},{4, 0b1111},{0, 0b0}}, + + {{3, 0b000},{3, 0b001},{2, 0b01},{1, 0b1},{0, 0b0},{0, 0b0}}, + {{2, 0b00},{3, 0b010},{1, 0b1},{3, 0b011},{0, 0b0},{0, 0b0}}, + {{2, 0b00},{2, 0b01},{2, 0b10},{2, 0b11},{0, 0b0},{0, 0b0}}, + {{1, 0b0},{3, 0b100},{2, 0b11},{3, 0b101},{0, 0b0},{0, 0b0}}, + {{1, 0b0},{2, 0b10},{3, 0b110},{3, 0b111},{0, 0b0},{0, 0b0}}, + {{1, 0b0},{2, 0b10},{2, 0b11},{0, 0b0},{0, 0b0},{0, 0b0}}, + {{1, 0b0},{1, 0b1},{0, 0b0},{0, 0b0},{0, 0b0},{0, 0b0}}, + {{0, 0b0},{1, 0b1},{0, 0b0},{0, 0b0},{0, 0b0},{0, 0b0}} + }; + + ForwardInputStream inputStream(_packedData,0,_packedData.size(),true); + + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + size_t rawSize=rawData.size(); + ForwardOutputStream outputStream(rawData,0,rawSize); + + OptionalHuffmanDecoder<uint32_t> decoder; + { + uint32_t treeIndex=readBits(5); + for (uint32_t i=0;i<6;i++) + { + uint32_t length=treeDefinitions[treeIndex][i].length; + uint32_t code=treeDefinitions[treeIndex][i].code; + if (!length) + { + if (!i) decoder.setEmpty(0); + break; + } + if (treeIndex==17 && i<2) decoder.insert(HuffmanCode<uint32_t>{length,code,i+3}); + else decoder.insert(HuffmanCode<uint32_t>{length,code,i}); + } + } + + uint8_t dataMTFMap[256]; + createMTFMap(dataMTFMap); + + auto processOutput=[&](uint8_t value)->uint8_t + { + updateMTF(value,dataMTFMap); + return value; + }; + + while (!outputStream.eof()) + { + bool doCopy=true; + if (readBit()) + { + uint32_t count=readBits(2)+1; + if (count==4) + { + count=readBits(3)+4; + if (count==11) + { + count=readBits(4)+11; + if (count==25) + { + count=readBits(6)+25; + } else if (count==26) { + count=readBits(7)+89; + } + } + } + + count=std::min(count,uint32_t(rawSize-outputStream.getOffset())); + for (uint32_t i=0;i<count;i++) + { + uint32_t code=decoder.decode(readBit); + static const uint32_t symbolAdditions[6]={0,16,32,64,128,192}; + static const uint32_t symbolBits[6]={4,4,5,6,6,6}; + outputStream.writeByte(processOutput(decodeMTF(readBits(symbolBits[code])+symbolAdditions[code],dataMTFMap))); + } + doCopy=count!=216&&!outputStream.eof(); + } + if (doCopy) + { + uint32_t offset=uint32_t(outputStream.getOffset()); + uint32_t count,code; + if (!readBit()) + { + if (offset>=0x240?readBit():0) + { + code=4; + if (offset<0x340) code=7; + else if (offset<0x440) code=8; + else if (offset<0x640) code=9; + } else { + code=offset>=0x40?readBit():0; + count=2; + } + } else { + if (offset>=0x40?!readBit():0) + { + code=3; + if (offset<0x140) code=6; + } else if (offset>=0xa40?readBit():1) { + code=2; + } else { + code=5; + if (offset<0xb40) code=10; + else if (offset<0xc40) code=11; + else if (offset<0xe40) code=12; + else if (offset<0x1240) code=13; + else if (offset<0x1a40) code=14; + } + } + if (code>=2) + { + count=readBits(2)+3; + if (count==6) + { + count=readBits(3)+6; + if (count==11) + { + count=readBits(2)+11; + } else if (count==12) { + count=readBits(3)+15; + } else if (count==13) { + count=readBits(6)+23; + if (count==85) { + count=readBits(5)+85; + } else if (count==86) + count=readBits(7)+117; + } + } + } + static const uint32_t distanceAdditions[15]={ + 1,0x41,1,0x41,0x241,0xa41,0x41,0x241,0x241,0x241,0xa41,0xa41,0xa41,0xa41,0xa41}; + static const uint32_t distanceBits[15]={ + 6,8,6,9,11,13,8,8,9,10,8,9,10,11,12}; + uint32_t distance=readBits(distanceBits[code])+distanceAdditions[code]; + + count=std::min(count,uint32_t(rawSize-outputStream.getOffset())); + outputStream.copy(distance,count,0x20); + const uint8_t *block=outputStream.history(count); + for (uint32_t i=0;i<count;i++) + processOutput(block[i]); + } + } +} + +void PMDecompressor::decompressImplPM2(Buffer &rawData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + OptionalHuffmanDecoder<uint32_t> decoder; + OptionalHuffmanDecoder<uint32_t> distanceDecoder; + + auto createDecoder=[&]()->bool + { + decoder.reset(); + + // codes beyond 29 are going to fault anyway, but maybe they are not used? + uint8_t symbols[31]; + uint32_t numCodes=readBits(5); + uint32_t minLength=readBits(3); + + bool ret=(numCodes>=10)&&(numCodes!=29||minLength); + + if (!minLength) + { + if (!numCodes) throw DecompressionError(); + decoder.setEmpty(numCodes-1); + } else { + uint32_t codeLength=readBits(3); + for (uint32_t i=0;i<numCodes;i++) + { + uint32_t value=readBits(codeLength); + symbols[i]=value?minLength+value-1:0; + } + decoder.createOrderlyHuffmanTable(symbols,numCodes); + } + return ret; + }; + + auto createDistanceDecoder=[&](uint32_t numCodes) + { + distanceDecoder.reset(); + + uint8_t distances[8]; + + uint32_t last=0,total=0; + for (uint32_t i=0;i<numCodes;i++) + { + uint32_t length=readBits(3); + distances[i]=length; + if (length) + { + total++; + last=i; + } + } + if (!total) + { + throw DecompressionError(); + } else if (total==1) { + distanceDecoder.setEmpty(last); + } else { + distanceDecoder.createOrderlyHuffmanTable(distances,numCodes); + } + }; + + uint8_t dataMTFMap[256]; + createMTFMap(dataMTFMap); + + bool distanceTreeRequired; + auto processOutput=[&](size_t offset,uint8_t value)->uint8_t + { + offset++; // we are interested offset after the update + updateMTF(value,dataMTFMap); + if (!(offset&0x3ffU)) + { + switch (offset>>10) + { + case 1: + if (distanceTreeRequired) createDistanceDecoder(6); + break; + + case 2: + if (distanceTreeRequired) createDistanceDecoder(7); + break; + + case 4: + if (readBit()) distanceTreeRequired=createDecoder(); + if (distanceTreeRequired) createDistanceDecoder(8); + break; + + default: + if (!(offset&0xfffU) && (offset>>12)>=2) + { + if (readBit()) + { + distanceTreeRequired=createDecoder(); + if (distanceTreeRequired) createDistanceDecoder(8); + } + } + break; + } + } + return value; + }; + + readBit(); // ignore first bit + distanceTreeRequired=createDecoder(); + if (distanceTreeRequired) createDistanceDecoder(5); + while (!outputStream.eof()) + { + uint32_t code=decoder.decode(readBit); + if (code<8) + { + static const uint32_t symbolAdditions[8]={0,8,16,32,64,96,128,192}; + static const uint32_t symbolBits[8]={3,3,4,5,5,5,6,6}; + outputStream.writeByte(processOutput(outputStream.getOffset(),decodeMTF(readBits(symbolBits[code])+symbolAdditions[code],dataMTFMap))); + } else { + code-=8; + + uint32_t count; + if (code<15) + { + count=code+2; + } else { + if (code>=21) throw DecompressionError(); + static const uint32_t countAdditions[6]={17,25,33,65,129,256}; + static const uint32_t countBits[6]={3,3,5,6,7,0}; + count=readBits(countBits[code-15])+countAdditions[code-15]; + } + + uint32_t distance; + if (!code) + { + distance=readBits(6)+1; + } else if (code<20) { + uint32_t tmp=distanceDecoder.decode(readBit); + if (!tmp) distance=readBits(6)+1; + else distance=readBits(tmp+5)+(1<<(tmp+5))+1; + } else distance=1; + + outputStream.copy(distance,count,0x20); + const uint8_t *block=outputStream.history(count); + size_t offset=outputStream.getOffset()-count; + for (uint32_t i=0;i<count;i++) + processOutput(offset+i,block[i]); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/PMDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/PMDecompressor.hpp new file mode 100644 index 00000000..95340d2a --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Lzh/PMDecompressor.hpp @@ -0,0 +1,39 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef PMDECOMPRESSOR_HPP +#define PMDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class PMDecompressor : public Decompressor +{ +public: + PMDecompressor(const Buffer &packedData,uint32_t version); + virtual ~PMDecompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + +private: + static uint8_t decodeMTF(uint8_t value,uint8_t map[]); + static void updateMTF(uint8_t value,uint8_t map[]); + static void createMTFMap(uint8_t map[]); + + void decompressImplPM1(Buffer &rawData,bool verify); + void decompressImplPM2(Buffer &rawData,bool verify); + + const Buffer &_packedData; + + uint32_t _version; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/MASHDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/MASHDecompressor.cpp new file mode 100644 index 00000000..247f0e3f --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/MASHDecompressor.cpp @@ -0,0 +1,121 @@ +/* Copyright (C) Teemu Suutari */ + +#include "MASHDecompressor.hpp" +#include "HuffmanDecoder.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool MASHDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("MASH"); +} + +std::shared_ptr<XPKDecompressor> MASHDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<MASHDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +MASHDecompressor::MASHDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +MASHDecompressor::~MASHDecompressor() +{ + // nothing needed +} + +const std::string &MASHDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-MASH: LZRW-compressor"; + return name; +} + +void MASHDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + size_t rawSize=rawData.size(); + ForwardOutputStream outputStream(rawData,0,rawSize); + + HuffmanDecoder<uint32_t> litDecoder + { + HuffmanCode<uint32_t>{1,0b000000,0}, + HuffmanCode<uint32_t>{2,0b000010,1}, + HuffmanCode<uint32_t>{3,0b000110,2}, + HuffmanCode<uint32_t>{4,0b001110,3}, + HuffmanCode<uint32_t>{5,0b011110,4}, + HuffmanCode<uint32_t>{6,0b111110,5}, + HuffmanCode<uint32_t>{6,0b111111,6} + }; + + while (!outputStream.eof()) + { + uint32_t litLength=litDecoder.decode(readBit); + if (litLength==6) + { + uint32_t litBits; + for (litBits=1;litBits<=17;litBits++) if (!readBit()) break; + if (litBits==17) throw Decompressor::DecompressionError(); + litLength=readBits(litBits)+(1<<litBits)+4; + } + + for (uint32_t i=0;i<litLength;i++) outputStream.writeByte(readByte()); + + uint32_t count,distance; + + auto readDistance=[&]() + { + uint32_t tableIndex=readBits(3); + static const uint8_t distanceBits[8]={5,7,9,10,11,12,13,14}; + static const uint32_t distanceAdditions[8]={0,0x20,0xa0,0x2a0,0x6a0,0xea0,0x1ea0,0x3ea0}; + distance=readBits(distanceBits[tableIndex])+distanceAdditions[tableIndex]; + }; + + if (readBit()) + { + uint32_t countBits; + for (countBits=1;countBits<=16;countBits++) if (!readBit()) break; + if (countBits==16) throw Decompressor::DecompressionError(); + count=readBits(countBits)+(1<<countBits)+2; + readDistance(); + } else { + if (readBit()) + { + readDistance(); + count=3; + } else { + distance=readBits(9); + count=2; + } + } + // hacks to make it work + if (!distance && outputStream.eof()) break; + // zero distance when we are at the end of the stream... + // there seems to be almost systematic extra one byte at the end of the stream... + count=std::min(count,uint32_t(rawSize-outputStream.getOffset())); + outputStream.copy(distance,count); + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/MASHDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/MASHDecompressor.hpp new file mode 100644 index 00000000..4e166c97 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/MASHDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef MASHDECOMPRESSOR_HPP +#define MASHDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class MASHDecompressor : public XPKDecompressor +{ +public: + MASHDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~MASHDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/MMCMPDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/MMCMPDecompressor.cpp new file mode 100644 index 00000000..49d8cbf2 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/MMCMPDecompressor.cpp @@ -0,0 +1,254 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstring> + +#include "MMCMPDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/OverflowCheck.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool MMCMPDecompressor::detectHeader(uint32_t hdr) noexcept +{ + return hdr==FourCC("ziRC"); +} + +std::shared_ptr<Decompressor> MMCMPDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) +{ + return std::make_shared<MMCMPDecompressor>(packedData,exactSizeKnown,verify); +} + +MMCMPDecompressor::MMCMPDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : + _packedData(packedData) +{ + if (!detectHeader(packedData.readBE32(0)) || packedData.readBE32(4U)!=FourCC("ONia") || + packedData.readLE16(8U)!=14U || packedData.size()<24U) + throw InvalidFormatError(); + _version=packedData.readLE16(10U); + _blocks=packedData.readLE16(12U); + _blocksOffset=packedData.readLE32(18U); + _rawSize=packedData.readLE32(14U); + if (OverflowCheck::sum(_blocksOffset,uint32_t(_blocks)*4U)>packedData.size()) + throw InvalidFormatError(); + + _packedSize=0; + for (uint32_t i=0;i<_blocks;i++) + { + uint32_t blockAddr=packedData.readLE32(OverflowCheck::sum(_blocksOffset,i*4U)); + if (OverflowCheck::sum(blockAddr,20U)>=packedData.size()) + throw InvalidFormatError(); + uint32_t blockSize=packedData.readLE32(blockAddr+4U)+uint32_t(packedData.readLE16(blockAddr+12U))*8U+20U; + _packedSize=std::max(_packedSize,OverflowCheck::sum(blockAddr,blockSize)); + } + if (_packedSize>packedData.size()) + throw InvalidFormatError(); +} + +MMCMPDecompressor::~MMCMPDecompressor() +{ + // nothing needed +} + +const std::string &MMCMPDecompressor::getName() const noexcept +{ + static std::string name="MMCMP: Music Module Compressor"; + return name; +} + +size_t MMCMPDecompressor::getPackedSize() const noexcept +{ + return _packedSize; +} + +size_t MMCMPDecompressor::getRawSize() const noexcept +{ + return _rawSize; +} + +void MMCMPDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + if (rawData.size()<_rawSize) throw DecompressionError(); + // MMCMP allows gaps in data. Although not used in practice still we memset before decompressing to be sure + std::memset(rawData.data(),0,rawData.size()); + + uint8_t *rawDataPtr=rawData.data(); + for (uint32_t i=0;i<_blocks;i++) + { + uint32_t blockAddr=_packedData.readLE32(_blocksOffset+i*4U); + + uint32_t unpackedBlockSize=_packedData.readLE32(blockAddr); + uint32_t packedBlockSize=_packedData.readLE32(blockAddr+4U); + uint32_t fileChecksum=_packedData.readLE32(blockAddr+8U); + uint32_t subBlocks=_packedData.readLE16(blockAddr+12U); + uint16_t flags=_packedData.readLE16(blockAddr+14U); + + uint32_t packTableSize=_packedData.readLE16(blockAddr+16U); + if (packTableSize>packedBlockSize) + throw DecompressionError(); + uint16_t bitCount=_packedData.readLE16(blockAddr+18U); + + ForwardInputStream inputStream(_packedData,OverflowCheck::sum(blockAddr,subBlocks*8U,20U,packTableSize),OverflowCheck::sum(blockAddr,subBlocks*8U,20U,packedBlockSize)); + LSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + + uint32_t currentSubBlock=0; + uint32_t outputOffset=0,outputSize=0; + auto readNextSubBlock=[&]() + { + if (currentSubBlock>=subBlocks) + throw DecompressionError(); + outputOffset=_packedData.readLE32(blockAddr+currentSubBlock*8U+20U); + outputSize=_packedData.readLE32(blockAddr+currentSubBlock*8U+24U); + if (OverflowCheck::sum(outputOffset,outputSize)>_rawSize) + throw DecompressionError(); + currentSubBlock++; + }; + + uint32_t checksum=0,checksumPartial=0,checksumRot=0; + auto writeByte=[&](uint8_t value,bool allowOverrun=false) + { + while (!outputSize) + { + if (allowOverrun && currentSubBlock>=subBlocks) return; + readNextSubBlock(); + } + outputSize--; + rawDataPtr[outputOffset++]=value; + if (verify) + { + if (_version>=0x1310) + { + checksum^=value; + checksum=(checksum<<1)|(checksum>>31); + } else { + checksumPartial|=((uint32_t)value)<<checksumRot; + checksumRot+=8U; + if (checksumRot==32U) + { + checksum^=checksumPartial; + checksumPartial=0; + checksumRot=0; + } + } + } + }; + + // flags are + // 0 = compressed + // 1 = delta mode + // 2 = 16 bit mode + // 8 = stereo + // 9 = abs16 + // 10 = endian + // flags do not combine nicely + // no compress - no other flags + // compressed 8 bit - only delta (and presumably stereo matters) + // compressed 16 bit - all flags matter + if (!(flags&0x1U)) + { + // not compressed + for (uint32_t j=0;j<packedBlockSize;j++) + writeByte(inputStream.readByte()); + } else if (!(flags&0x4U)) { + // 8 bit compression + + // in case the bit-count is not enough to store a value, symbol at the end + // of the codemap is created and this marks as a new bitCount + static const uint8_t valueThresholds[16]={0x1U, 0x3U, 0x7U, 0xfU,0x1eU,0x3cU,0x78U,0xf8U}; + static const uint8_t extraBits[8]={3,3,3,3, 2,1,0,0}; + + if (bitCount>=8) + throw DecompressionError(); + uint8_t oldValue[2]={0,0}; + uint32_t chIndex=0; + const uint8_t *tablePtr=&_packedData[blockAddr+subBlocks*8U+20U]; + for (uint32_t j=0;j<unpackedBlockSize;) + { + uint8_t value=readBits(bitCount+1); + if (value>=valueThresholds[bitCount]) + { + uint32_t newBitCount=readBits(extraBits[bitCount])+((value-valueThresholds[bitCount])<<extraBits[bitCount]); + if (bitCount!=newBitCount) + { + bitCount=newBitCount&0x7U; + continue; + } + value=0xf8U+readBits(3U); + if (value==0xffU && readBits(1U)) break; + } + if (value>=packTableSize) + throw DecompressionError(); + value=tablePtr[value]; + if (flags&0x2U) + { + // delta + value+=oldValue[chIndex]; + oldValue[chIndex]=value; + if (flags&0x100U) chIndex^=1U; // stereo + } + writeByte(value); + j++; + } + } else { + // 16 bit compression + + // shameless copy-paste from 8-bit variant, with minor changes + static const uint16_t valueThresholds[16]={ + 0x1U, 0x3U, 0x7U, 0xfU, 0x1eU, 0x3cU, 0x78U, 0xf0U, + 0x1f0U, 0x3f0U, 0x7f0U, 0xff0U,0x1ff0U,0x3ff0U,0x7ff0U,0xfff0U + }; + static const uint8_t extraBits[16]={4,4,4,4, 3,2,1,0, 0,0,0,0, 0,0,0,0}; + + if (bitCount>=16) + throw DecompressionError(); + int16_t oldValue[2]={0,0}; + uint32_t chIndex=0; + for (uint32_t j=0;j<unpackedBlockSize;) + { + int32_t value=readBits(bitCount+1); + if (value>=valueThresholds[bitCount]) + { + uint32_t newBitCount=readBits(extraBits[bitCount])+((value-valueThresholds[bitCount])<<extraBits[bitCount]); + if (bitCount!=newBitCount) + { + bitCount=newBitCount&0xfU; + continue; + } + value=0xfff0U+readBits(4U); + if (value==0xffffU && readBits(1U)) break; + } + if (value&1U) value=-value-1; + value>>=1; + if (flags&0x2U) + { + // delta + value+=oldValue[chIndex]; + oldValue[chIndex]=value; + if (flags&0x100U) chIndex^=1U; // stereo + } else if (!(flags&0x200U)) value^=0x8000U; // abs16 + if (flags&0x400U) + { + // big ending + writeByte(value>>8U); + writeByte(value,true); + } else { + // little endian + writeByte(value); + writeByte(value>>8U,true); + } + j+=2; + } + } + if (verify && checksum!=fileChecksum) + throw VerificationError(); + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/MMCMPDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/MMCMPDecompressor.hpp new file mode 100644 index 00000000..8c9a497f --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/MMCMPDecompressor.hpp @@ -0,0 +1,39 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef MMCMPDECOMPRESSOR_HPP +#define MMCMPDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class MMCMPDecompressor : public Decompressor +{ +public: + MMCMPDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); + + virtual ~MMCMPDecompressor(); + + virtual const std::string &getName() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + virtual size_t getRawSize() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + + static bool detectHeader(uint32_t hdr) noexcept; + static std::shared_ptr<Decompressor> create(const Buffer &packedData,bool exactSizeKnown,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _packedSize=0; + uint32_t _rawSize=0; + uint32_t _blocksOffset=0; + uint32_t _blocks=0; + uint16_t _version=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/NONEDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/NONEDecompressor.cpp new file mode 100644 index 00000000..22ed94f1 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/NONEDecompressor.cpp @@ -0,0 +1,47 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstring> + +#include "NONEDecompressor.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool NONEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("NONE"); +} + +std::shared_ptr<XPKDecompressor> NONEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<NONEDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +NONEDecompressor::NONEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +NONEDecompressor::~NONEDecompressor() +{ + // nothing needed +} + +const std::string &NONEDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-NONE: Null compressor"; + return name; +} + +void NONEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + if (rawData.size()!=_packedData.size()) throw Decompressor::DecompressionError(); + + std::memcpy(rawData.data(),_packedData.data(),_packedData.size()); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/NONEDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/NONEDecompressor.hpp new file mode 100644 index 00000000..978e6a6d --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/NONEDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef NONEDECOMPRESSOR_HPP +#define NONEDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class NONEDecompressor : public XPKDecompressor +{ +public: + NONEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~NONEDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/NUKEDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/NUKEDecompressor.cpp new file mode 100644 index 00000000..25cebbc9 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/NUKEDecompressor.cpp @@ -0,0 +1,131 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstring> + +#include "NUKEDecompressor.hpp" +#include "DLTADecode.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool NUKEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("NUKE") || hdr==FourCC("DUKE"); +} + +std::shared_ptr<XPKDecompressor> NUKEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<NUKEDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +NUKEDecompressor::NUKEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); + if (hdr==FourCC("DUKE")) _isDUKE=true; +} + +NUKEDecompressor::~NUKEDecompressor() +{ + // nothing needed +} + +const std::string &NUKEDecompressor::getSubName() const noexcept +{ + static std::string nameN="XPK-NUKE: LZ77-compressor"; + static std::string nameD="XPK-DUKE: LZ77-compressor with delta encoding"; + return (_isDUKE)?nameD:nameN; +} + +void NUKEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + // there are 2 streams, reverse stream for bytes and + // normal stream for bits, the bit stream is divided + // into single bit, 2 bit, 4 bit and random accumulator + ForwardInputStream forwardInputStream(_packedData,0,_packedData.size()); + BackwardInputStream backwardInputStream(_packedData,0,_packedData.size()); + forwardInputStream.link(backwardInputStream); + backwardInputStream.link(forwardInputStream); + MSBBitReader<ForwardInputStream> bit1Reader(forwardInputStream); + MSBBitReader<ForwardInputStream> bit2Reader(forwardInputStream); + LSBBitReader<ForwardInputStream> bit4Reader(forwardInputStream); + MSBBitReader<ForwardInputStream> bitXReader(forwardInputStream); + auto readBit=[&]()->uint32_t + { + return bit1Reader.readBitsBE16(1); + }; + auto read2Bits=[&]()->uint32_t + { + return bit2Reader.readBitsBE16(2); + }; + auto read4Bits=[&]()->uint32_t + { + return bit4Reader.readBitsBE32(4); + }; + auto readBits=[&](uint32_t count)->uint32_t + { + return bitXReader.readBitsBE16(count); + }; + auto readByte=[&]()->uint8_t + { + return backwardInputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + for (;;) + { + if (!readBit()) + { + uint32_t count=0; + if (readBit()) + { + count=1; + } else { + uint32_t tmp; + do { + tmp=read2Bits(); + if (tmp) count+=5-tmp; + else count+=3; + } while (!tmp); + } + for (uint32_t i=0;i<count;i++) outputStream.writeByte(readByte()); + } + if (outputStream.eof()) break; + uint32_t distanceIndex=read4Bits(); + static const uint8_t distanceBits[16]={ + 4,6,8,9, + 4,7,9,11,13,14, + 5,7,9,11,13,14}; + static const uint32_t distanceAdditions[16]={ + 0,0x10,0x50,0x150, + 0,0x10,0x90,0x290,0xa90,0x2a90, + 0,0x20,0xa0,0x2a0,0xaa0,0x2aa0}; + uint32_t distance=readBits(distanceBits[distanceIndex])+distanceAdditions[distanceIndex]; + uint32_t count=(distanceIndex<4)?2:(distanceIndex<10)?3:0; + if (!count) + { + count=read2Bits(); + if (!count) + { + count=3+3; + uint32_t tmp; + do { + tmp=read4Bits(); + if (tmp) count+=16-tmp; + else count+=15; + } while (!tmp); + } else count=3+4-count; + } + outputStream.copy(distance,count); + } + if (_isDUKE) + DLTADecode::decode(rawData,rawData,0,rawData.size()); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/NUKEDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/NUKEDecompressor.hpp new file mode 100644 index 00000000..d3a540ce --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/NUKEDecompressor.hpp @@ -0,0 +1,33 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef NUKEDECOMPRESSOR_HPP +#define NUKEDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class NUKEDecompressor : public XPKDecompressor +{ +public: + NUKEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~NUKEDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + bool _isDUKE=false; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/OutputStream.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/OutputStream.cpp new file mode 100644 index 00000000..c2bad30a --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/OutputStream.cpp @@ -0,0 +1,124 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstring> + +#include <algorithm> + +#include "OutputStream.hpp" +// for exceptions +#include "Decompressor.hpp" +#include "common/Common.hpp" +#include "common/OverflowCheck.hpp" + + +namespace ancient::internal +{ + +ForwardOutputStream::ForwardOutputStream(Buffer &buffer,size_t startOffset,size_t endOffset) : + _bufPtr(buffer.data()), + _startOffset(startOffset), + _currentOffset(startOffset), + _endOffset(endOffset) +{ + if (_startOffset>_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); +} + +ForwardOutputStream::~ForwardOutputStream() +{ + // nothing needed +} + +void ForwardOutputStream::writeByte(uint8_t value) +{ + if (_currentOffset>=_endOffset) throw Decompressor::DecompressionError(); + _bufPtr[_currentOffset++]=value; +} + +uint8_t ForwardOutputStream::copy(size_t distance,size_t count) +{ + if (!distance || OverflowCheck::sum(_startOffset,distance)>_currentOffset || OverflowCheck::sum(_currentOffset,count)>_endOffset) throw Decompressor::DecompressionError(); + uint8_t ret=0; + for (size_t i=0;i<count;i++,_currentOffset++) + ret=_bufPtr[_currentOffset]=_bufPtr[_currentOffset-distance]; + return ret; +} + +uint8_t ForwardOutputStream::copy(size_t distance,size_t count,const Buffer &prevBuffer) +{ + if (!distance || OverflowCheck::sum(_currentOffset,count)>_endOffset) throw Decompressor::DecompressionError(); + size_t prevCount=0; + uint8_t ret=0; + if (OverflowCheck::sum(_startOffset,distance)>_currentOffset) + { + size_t prevSize=prevBuffer.size(); + if (_startOffset+distance>_currentOffset+prevSize) throw Decompressor::DecompressionError(); + size_t prevDist=_startOffset+distance-_currentOffset; + prevCount=std::min(count,prevDist); + const uint8_t *prev=&prevBuffer[prevSize-prevDist]; + for (size_t i=0;i<prevCount;i++,_currentOffset++) + ret=_bufPtr[_currentOffset]=prev[i]; + } + for (size_t i=prevCount;i<count;i++,_currentOffset++) + ret=_bufPtr[_currentOffset]=_bufPtr[_currentOffset-distance]; + return ret; +} + +uint8_t ForwardOutputStream::copy(size_t distance,size_t count,uint8_t defaultChar) +{ + if (!distance || OverflowCheck::sum(_currentOffset,count)>_endOffset) throw Decompressor::DecompressionError(); + size_t prevCount=0; + uint8_t ret=0; + if (OverflowCheck::sum(_startOffset,distance)>_currentOffset) + { + prevCount=std::min(count,_startOffset+distance-_currentOffset); + for (size_t i=0;i<prevCount;i++,_currentOffset++) + ret=_bufPtr[_currentOffset]=defaultChar; + } + for (size_t i=prevCount;i<count;i++,_currentOffset++) + ret=_bufPtr[_currentOffset]=_bufPtr[_currentOffset-distance]; + return ret; +} + +const uint8_t *ForwardOutputStream::history(size_t distance) const +{ + if (distance>_currentOffset) throw Decompressor::DecompressionError(); + return &_bufPtr[_currentOffset-distance]; +} + +void ForwardOutputStream::produce(const uint8_t *src,size_t bytes) +{ + if (OverflowCheck::sum(_currentOffset,bytes)>_endOffset) throw Decompressor::DecompressionError(); + std::memcpy(&_bufPtr[_currentOffset],src,bytes); + _currentOffset+=bytes; +} + +BackwardOutputStream::BackwardOutputStream(Buffer &buffer,size_t startOffset,size_t endOffset) : + _bufPtr(buffer.data()), + _startOffset(startOffset), + _currentOffset(endOffset), + _endOffset(endOffset) +{ + if (_startOffset>_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); +} + +BackwardOutputStream::~BackwardOutputStream() +{ + // nothing needed +} + +void BackwardOutputStream::writeByte(uint8_t value) +{ + if (_currentOffset<=_startOffset) throw Decompressor::DecompressionError(); + _bufPtr[--_currentOffset]=value; +} + +uint8_t BackwardOutputStream::copy(size_t distance,size_t count) +{ + if (!distance || OverflowCheck::sum(_startOffset,count)>_currentOffset || OverflowCheck::sum(_currentOffset,distance)>_endOffset) throw Decompressor::DecompressionError(); + uint8_t ret=0; + for (size_t i=0;i<count;i++,--_currentOffset) + ret=_bufPtr[_currentOffset-1]=_bufPtr[_currentOffset+distance-1]; + return ret; +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/OutputStream.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/OutputStream.hpp new file mode 100644 index 00000000..abbb667e --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/OutputStream.hpp @@ -0,0 +1,61 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef OUTPUTSTREAM_HPP +#define OUTPUTSTREAM_HPP + +#include <cstddef> +#include <cstdint> + +#include "common/Buffer.hpp" + +namespace ancient::internal +{ + +class ForwardOutputStream +{ +public: + ForwardOutputStream(Buffer &buffer,size_t startOffset,size_t endOffset); + ~ForwardOutputStream(); + + void writeByte(uint8_t value); + + uint8_t copy(size_t distance,size_t count); + uint8_t copy(size_t distance,size_t count,const Buffer &prevBuffer); + uint8_t copy(size_t distance,size_t count,uint8_t defaultChar); + const uint8_t *history(size_t distance) const; + void produce(const uint8_t *src,size_t bytes); + + bool eof() const { return _currentOffset==_endOffset; } + size_t getOffset() const { return _currentOffset; } + size_t getEndOffset() const { return _endOffset; } + +private: + uint8_t *_bufPtr; + size_t _startOffset; + size_t _currentOffset; + size_t _endOffset; +}; + +class BackwardOutputStream +{ +public: + BackwardOutputStream(Buffer &buffer,size_t startOffset,size_t endOffset); + ~BackwardOutputStream(); + + void writeByte(uint8_t value); + + uint8_t copy(size_t distance,size_t count); + + bool eof() const { return _currentOffset==_startOffset; } + size_t getOffset() const { return _currentOffset; } + +private: + uint8_t *_bufPtr; + size_t _startOffset; + size_t _currentOffset; + size_t _endOffset; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/PPDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/PPDecompressor.cpp new file mode 100644 index 00000000..f050ea91 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/PPDecompressor.cpp @@ -0,0 +1,192 @@ +/* Copyright (C) Teemu Suutari */ + +#include "PPDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +PPDecompressor::PPState::PPState(uint32_t mode) : + _cachedMode(mode) +{ + // nothing needed +} + +PPDecompressor::PPState::~PPState() +{ + // nothing needed +} + +bool PPDecompressor::detectHeader(uint32_t hdr) noexcept +{ + return (hdr==FourCC("PP11") || hdr==FourCC("PP20")); +} + +bool PPDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("PWPK"); +} + +std::shared_ptr<Decompressor> PPDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) +{ + return std::make_shared<PPDecompressor>(packedData,exactSizeKnown,verify); +} + +std::shared_ptr<XPKDecompressor> PPDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<PPDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +PPDecompressor::PPDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : + _packedData(packedData) +{ + if (!exactSizeKnown || packedData.size()<0x10) + throw InvalidFormatError(); // no scanning support + _dataStart=_packedData.size()-4; + + uint32_t hdr=packedData.readBE32(0); + if (!detectHeader(hdr)) throw InvalidFormatError(); + uint32_t mode=packedData.readBE32(4); + if (mode!=0x9090909 && mode!=0x90a0a0a && mode!=0x90a0b0b && mode!=0x90a0c0c && mode!=0x90a0c0d) throw InvalidFormatError(); + for (uint32_t i=0;i<4;i++) + { + _modeTable[i]=mode>>24; + mode<<=8; + } + + uint32_t tmp=packedData.readBE32(_dataStart); + + _rawSize=tmp>>8; + _startShift=tmp&0xff; + if (!_rawSize || _startShift>=0x20 || + _rawSize>getMaxRawSize()) throw InvalidFormatError(); +} + +PPDecompressor::PPDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr) || packedData.size()<0x10) + throw InvalidFormatError(); + _dataStart=_packedData.size()-4; + + uint32_t mode; + if (state.get()) + { + mode=static_cast<PPState*>(state.get())->_cachedMode; + } else { + mode=packedData.readBE32(_dataStart); + if (mode>4) throw InvalidFormatError(); + state.reset(new PPState(mode)); + _dataStart-=4; + } + + static const uint32_t modeMap[5]={0x9090909,0x90a0a0a,0x90a0b0b,0x90a0c0c,0x90a0c0d}; + mode=modeMap[mode]; + for (uint32_t i=0;i<4;i++) + { + _modeTable[i]=mode>>24; + mode<<=8; + } + + uint32_t tmp=packedData.readBE32(_dataStart); + + _rawSize=tmp>>8; + _startShift=tmp&0xff; + if (!_rawSize || _startShift>=0x20 || _rawSize>getMaxRawSize()) + throw InvalidFormatError(); + + _isXPK=true; +} + +PPDecompressor::~PPDecompressor() +{ + // nothing needed +} + +const std::string &PPDecompressor::getName() const noexcept +{ + static std::string name="PP: PowerPacker"; + return name; +} + +const std::string &PPDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-PWPK: PowerPacker"; + return name; +} + +size_t PPDecompressor::getPackedSize() const noexcept +{ + return 0; +} + +size_t PPDecompressor::getRawSize() const noexcept +{ + return _rawSize; +} + +void PPDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + if (rawData.size()<_rawSize) throw DecompressionError(); + + BackwardInputStream inputStream(_packedData,_isXPK?0:8,_dataStart); + LSBBitReader<BackwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return rotateBits(bitReader.readBitsBE32(count),count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBitsBE32(1); + }; + readBits(_startShift); + + BackwardOutputStream outputStream(rawData,0,_rawSize); + + for (;;) + { + if (!readBit()) + { + uint32_t count=1; + // This does not make much sense I know. But it is what it is... + for (;;) + { + uint32_t tmp=readBits(2); + count+=tmp; + if (tmp<3) break; + } + for (uint32_t i=0;i<count;i++) outputStream.writeByte(readBits(8)); + } + if (outputStream.eof()) break; + uint32_t modeIndex=readBits(2); + uint32_t count,distance; + if (modeIndex==3) + { + distance=readBits(readBit()?_modeTable[modeIndex]:7)+1; + // ditto + count=5; + for (;;) + { + uint32_t tmp=readBits(3); + count+=tmp; + if (tmp<7) break; + } + } else { + count=modeIndex+2; + distance=readBits(_modeTable[modeIndex])+1; + } + outputStream.copy(distance,count); + } +} + +void PPDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + if (_rawSize!=rawData.size()) throw DecompressionError(); + decompressImpl(rawData,verify); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/PPDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/PPDecompressor.hpp new file mode 100644 index 00000000..5d8d6424 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/PPDecompressor.hpp @@ -0,0 +1,56 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef PPDECOMPRESSOR_HPP +#define PPDECOMPRESSOR_HPP + +#include "Decompressor.hpp" +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class PPDecompressor : public Decompressor, public XPKDecompressor +{ +private: + class PPState : public XPKDecompressor::State + { + public: + PPState(uint32_t mode); + virtual ~PPState(); + + uint32_t _cachedMode; + }; + +public: + PPDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); + PPDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + virtual ~PPDecompressor(); + + virtual const std::string &getName() const noexcept override final; + virtual const std::string &getSubName() const noexcept override final; + + virtual size_t getPackedSize() const noexcept override final; + virtual size_t getRawSize() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeader(uint32_t hdr) noexcept; + static bool detectHeaderXPK(uint32_t hdr) noexcept; + + static std::shared_ptr<Decompressor> create(const Buffer &packedData,bool exactSizeKnown,bool verify); + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + size_t _dataStart=0; + size_t _rawSize=0; + uint8_t _startShift=0; + uint8_t _modeTable[4]; + bool _isXPK=false; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/RAKEDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RAKEDecompressor.cpp new file mode 100644 index 00000000..1e07c13f --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RAKEDecompressor.cpp @@ -0,0 +1,147 @@ +/* Copyright (C) Teemu Suutari */ + +#include "RAKEDecompressor.hpp" +#include "HuffmanDecoder.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool RAKEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return (hdr==FourCC("FRHT") || hdr==FourCC("RAKE")); +} + +std::shared_ptr<XPKDecompressor> RAKEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<RAKEDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +RAKEDecompressor::RAKEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData), + _isRAKE(hdr==FourCC("RAKE")) +{ + if (!detectHeaderXPK(hdr) || packedData.size()<4) + throw Decompressor::InvalidFormatError(); + + _midStreamOffset=packedData.readBE16(2); + if (_midStreamOffset>=packedData.size()) throw Decompressor::InvalidFormatError(); +} + +RAKEDecompressor::~RAKEDecompressor() +{ + // nothing needed +} + +const std::string &RAKEDecompressor::getSubName() const noexcept +{ + static std::string nameFRHT="XPK-FRHT: LZ77-compressor"; + static std::string nameRAKE="XPK-RAKE: LZ77-compressor"; + return (_isRAKE)?nameRAKE:nameFRHT; +} + +void RAKEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + // 2 streams + // 1st: bit stream starting from _midStreamOffset(+1) going to packedSize + // 2nd: byte stream starting from _midStreamOffset going backwards to 4 + ForwardInputStream forwardInputStream(_packedData,_midStreamOffset+(_midStreamOffset&1),_packedData.size()); + BackwardInputStream backwardInputStream(_packedData,4,_midStreamOffset); + MSBBitReader<ForwardInputStream> bitReader(forwardInputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBitsBE32(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBitsBE32(1); + }; + auto readByte=[&]()->uint8_t + { + return backwardInputStream.readByte(); + }; + { + uint16_t tmp=_packedData.readBE16(0); + if (tmp>32) throw Decompressor::DecompressionError(); + const uint8_t *buf=forwardInputStream.consume(4); + uint32_t content=(uint32_t(buf[0])<<24)|(uint32_t(buf[1])<<16)| + (uint32_t(buf[2])<<8)|uint32_t(buf[3]); + bitReader.reset(content>>tmp,32-tmp); + } + + BackwardOutputStream outputStream(rawData,0,rawData.size()); + + HuffmanDecoder<uint32_t> lengthDecoder; + // is there some logic into this? + static const uint8_t decTable[255][2]={ + { 1,0x01},{ 3,0x03},{ 5,0x05},{ 6,0x09},{ 7,0x0c},{ 9,0x13},{12,0x34},{18,0xc0}, + {18,0xc2},{18,0xc3},{18,0xc6},{16,0x79},{18,0xc7},{18,0xd6},{18,0xd7},{18,0xd8}, + {17,0xa8},{17,0x92},{17,0x8a},{17,0x82},{16,0x6c},{17,0x94},{18,0xda},{18,0xca}, + {16,0x7b},{13,0x36},{13,0x39},{13,0x48},{14,0x49},{14,0x50},{15,0x62},{15,0x5e}, + {16,0x6f},{17,0x83},{17,0x87},{15,0x56},{11,0x21},{12,0x31},{13,0x38},{13,0x3d}, + { 8,0x0f},{ 4,0x04},{ 6,0x08},{10,0x1c},{12,0x27},{13,0x42},{13,0x3a},{12,0x30}, + {12,0x32},{ 9,0x16},{ 8,0x11},{ 7,0x0b},{ 5,0x06},{10,0x19},{10,0x1a},{10,0x18}, + {11,0x26},{17,0x98},{17,0x99},{17,0x9b},{17,0x9e},{17,0x9f},{17,0xa6},{16,0x73}, + {17,0x7f},{17,0x81},{17,0x84},{17,0x85},{15,0x5d},{14,0x4d},{14,0x4f},{13,0x45}, + {13,0x3c},{ 9,0x17},{10,0x1d},{12,0xff},{13,0x41},{17,0x8c},{18,0xaa},{19,0xdb}, + {19,0xdc},{16,0x77},{15,0x63},{16,0x7c},{16,0x76},{16,0x71},{16,0x7d},{12,0x2c}, + {13,0x3b},{16,0x7a},{16,0x75},{15,0x55},{15,0x60},{16,0x74},{17,0xa4},{18,0xab}, + {18,0xac},{ 7,0x0a},{ 6,0x07},{ 9,0x15},{11,0x20},{11,0x24},{10,0x1b},{ 8,0x10}, + { 9,0x12},{12,0x33},{14,0x4b},{15,0x53},{19,0xdd},{19,0xde},{18,0xad},{19,0xdf}, + {19,0xe0},{18,0xae},{17,0x88},{18,0xaf},{19,0xe1},{19,0xe2},{13,0x37},{12,0x2e}, + {18,0xb0},{18,0xb1},{19,0xe3},{19,0xe4},{18,0xb2},{18,0xb3},{19,0xe5},{19,0xe6}, + {19,0xe7},{19,0xe8},{18,0xb4},{17,0x9a},{18,0xb5},{18,0xb6},{18,0xb7},{19,0xe9}, + {19,0xea},{18,0xb8},{19,0xeb},{19,0xec},{19,0xed},{19,0xee},{18,0xb9},{19,0xef}, + {19,0xf0},{18,0xbb},{18,0xbc},{19,0xf1},{19,0xf2},{18,0xbd},{18,0xbe},{19,0xf3}, + {19,0xf4},{18,0xbf},{18,0xc1},{19,0xf5},{19,0xf6},{18,0xc4},{18,0xc5},{17,0x95}, + {18,0xc8},{18,0xc9},{19,0xf7},{19,0xf8},{18,0xcb},{18,0xcc},{19,0xf9},{19,0xfa}, + {18,0xcd},{18,0xce},{17,0x96},{18,0xcf},{18,0xd0},{19,0xfb},{19,0xfc},{18,0xd1}, + {18,0xd2},{18,0xd3},{17,0x9c},{17,0x9d},{18,0xd4},{18,0xd5},{17,0xa0},{17,0xa1}, + {17,0xa2},{17,0xa3},{17,0xa5},{19,0xfd},{19,0xfe},{18,0xd9},{17,0xa7},{16,0x66}, + {15,0x54},{15,0x57},{16,0x6b},{16,0x68},{14,0x4c},{14,0x4e},{12,0x28},{11,0x23}, + { 8,0x0e},{ 7,0x0d},{10,0x1f},{13,0x47},{15,0x64},{15,0x58},{15,0x59},{15,0x5a}, + {12,0x29},{13,0x3e},{15,0x5f},{17,0x8e},{18,0xba},{18,0xa9},{16,0x70},{14,0x4a}, + {12,0x2a},{ 9,0x14},{11,0x22},{12,0x2f},{16,0x7e},{16,0x67},{16,0x69},{16,0x65}, + {15,0x51},{16,0x78},{16,0x6a},{13,0x46},{11,0x25},{16,0x72},{16,0x6e},{15,0x5b}, + {15,0x61},{15,0x52},{13,0x40},{13,0x43},{13,0x44},{13,0x3f},{15,0x5c},{17,0x93}, + {17,0x80},{17,0x8d},{17,0x8b},{17,0x86},{17,0x89},{17,0x97},{17,0x8f},{17,0x90}, + {17,0x91},{16,0x6d},{12,0x2b},{12,0x2d},{12,0x35},{10,0x1e},{ 3,0x02}}; + + uint32_t hufCode=0; + for (auto &it: decTable) + { + lengthDecoder.insert(HuffmanCode<uint32_t>{it[0],hufCode>>(32-it[0]),it[1]}); + hufCode+=1<<(32-it[0]); + } + + while (!outputStream.eof()) + { + if (!readBit()) + { + outputStream.writeByte(readByte()); + } else { + uint32_t count=lengthDecoder.decode(readBit); + count+=2; + + uint32_t distance; + if (!readBit()) + { + distance=uint32_t(readByte())+1; + } else { + if (!readBit()) + { + distance=((readBits(3)<<8)|uint32_t(readByte()))+0x101; + } else { + distance=((readBits(6)<<8)|uint32_t(readByte()))+0x901; + } + } + outputStream.copy(distance,count); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/RAKEDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RAKEDecompressor.hpp new file mode 100644 index 00000000..562aafea --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RAKEDecompressor.hpp @@ -0,0 +1,34 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef RAKEDECOMPRESSOR_HPP +#define RAKEDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class RAKEDecompressor : public XPKDecompressor +{ +public: + RAKEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~RAKEDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + bool _isRAKE; + size_t _midStreamOffset=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/RDCNDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RDCNDecompressor.cpp new file mode 100644 index 00000000..ec2e75a1 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RDCNDecompressor.cpp @@ -0,0 +1,101 @@ +/* Copyright (C) Teemu Suutari */ + +#include "RDCNDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool RDCNDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("RDCN"); +} + +std::shared_ptr<XPKDecompressor> RDCNDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<RDCNDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +RDCNDecompressor::RDCNDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +RDCNDecompressor::~RDCNDecompressor() +{ + // nothing needed +} + +const std::string &RDCNDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-RDCN: Ross data compression"; + return name; +} + +void RDCNDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBit=[&]()->uint32_t + { + return bitReader.readBitsBE16(1); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + while (!outputStream.eof()) + { + if (!readBit()) + { + outputStream.writeByte(readByte()); + } else { + uint8_t tmp=readByte(); + uint32_t count=tmp&0xf; + uint32_t code=tmp>>4; + uint32_t distance=0; + uint8_t repeatChar=0; + bool doRLE=false; + switch (code) + { + case 0: + repeatChar=readByte(); + count+=3; + doRLE=true; + break; + + case 1: + count=(count|(uint32_t(readByte())<<4))+19; + repeatChar=readByte(); + doRLE=true; + break; + + case 2: + distance=(count|(uint32_t(readByte())<<4))+3; + count=uint32_t(readByte())+16; + break; + + default: /* 3 to 15 */ + distance=(count|(uint32_t(readByte())<<4))+3; + count=code; + break; + } + if (doRLE) + { + for (uint32_t i=0;i<count;i++) outputStream.writeByte(repeatChar); + } else { + outputStream.copy(distance,count); + } + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/RDCNDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RDCNDecompressor.hpp new file mode 100644 index 00000000..da40c8dc --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RDCNDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef RDCNDECOMPRESSOR_HPP +#define RDCNDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class RDCNDecompressor : public XPKDecompressor +{ +public: + RDCNDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~RDCNDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/RLENDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RLENDecompressor.cpp new file mode 100644 index 00000000..6b1b0209 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RLENDecompressor.cpp @@ -0,0 +1,62 @@ +/* Copyright (C) Teemu Suutari */ + +#include "RLENDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool RLENDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("RLEN"); +} + +std::shared_ptr<XPKDecompressor> RLENDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<RLENDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +RLENDecompressor::RLENDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +RLENDecompressor::~RLENDecompressor() +{ + // nothing needed +} + +const std::string &RLENDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-RLEN: RLE-compressor"; + return name; +} + +void RLENDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + while (!outputStream.eof()) + { + uint32_t count=uint32_t(inputStream.readByte()); + if (count<128) + { + if (!count) throw Decompressor::DecompressionError(); // lets have this as error... + for (uint32_t i=0;i<count;i++) outputStream.writeByte(inputStream.readByte()); + } else { + // I can see from different implementations that count=0x80 is buggy... + // lets try to have it more or less correctly here + count=256-count; + uint8_t ch=inputStream.readByte(); + for (uint32_t i=0;i<count;i++) outputStream.writeByte(ch); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/RLENDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RLENDecompressor.hpp new file mode 100644 index 00000000..b5653e36 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RLENDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef RLENDECOMPRESSOR_HPP +#define RLENDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class RLENDecompressor : public XPKDecompressor +{ +public: + RLENDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~RLENDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/RNCDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RNCDecompressor.cpp new file mode 100644 index 00000000..83d82dfd --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RNCDecompressor.cpp @@ -0,0 +1,457 @@ +/* Copyright (C) Teemu Suutari */ + +#include <algorithm> + +#include "RNCDecompressor.hpp" +#include "HuffmanDecoder.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/CRC16.hpp" +#include "common/OverflowCheck.hpp" +#include "common/Common.hpp" + +// This allows decompression of pc compressed files from unonfficial (and unpatched) compressor +// PC games do not need chunk count, and are happy to read these files. +// Official tools put it and amiga decompressors require it +#define ALLOW_MISSING_CHUNKS 1 + +namespace ancient::internal +{ + +bool RNCDecompressor::detectHeader(uint32_t hdr) noexcept +{ + return hdr==FourCC("RNC\001") || hdr==FourCC("RNC\002"); +} + +std::shared_ptr<Decompressor> RNCDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) +{ + return std::make_shared<RNCDecompressor>(packedData,verify); +} + +RNCDecompressor::RNCDecompressor(const Buffer &packedData,bool verify) : + _packedData(packedData) +{ + uint32_t hdr=packedData.readBE32(0); + _rawSize=packedData.readBE32(4); + _packedSize=packedData.readBE32(8); + if (!_rawSize || !_packedSize || + _rawSize>getMaxRawSize() || _packedSize>getMaxPackedSize()) throw InvalidFormatError(); + + bool verified=false; + if (hdr==FourCC("RNC\001")) + { + // now detect between old and new version + // since the old and the new version share the same id, there is no foolproof way + // to tell them apart. It is easier to prove that it is not something by finding + // specific invalid bitstream content. + + // well, this is silly though but lets assume someone has made old format RNC1 with total size less than 19 + if (packedData.size()<19) + { + _ver=Version::RNC1Old; + } else { + uint8_t newStreamStart=packedData.read8(18); + uint8_t oldStreamStart=packedData.read8(_packedSize+11); + + // Check that stream starts with a literal(s) + if (!(oldStreamStart&0x80)) + _ver=Version::RNC1New; + + // New stream have two bits in start as a filler on new stream. Those are always 0 + // (although this is not strictly mandated) + // + + // Even though it is possible to make new RNC1 stream which starts with zero literal table size, + // it is extremely unlikely + else if ((newStreamStart&3) || !(newStreamStart&0x7c)) + _ver=Version::RNC1Old; + + // now the last resort: check CRC. + else if (_packedData.size()>=_packedSize+18 && CRC16(_packedData,18,_packedSize,0)==packedData.readBE16(14)) + { + _ver=Version::RNC1New; + verified=true; + } else _ver=Version::RNC1Old; + } + } else if (hdr==FourCC("RNC\002")) { + _ver=Version::RNC2; + } else throw InvalidFormatError(); + + uint32_t hdrSize=(_ver==Version::RNC1Old)?12:18; + if (OverflowCheck::sum(_packedSize,hdrSize)>packedData.size()) throw InvalidFormatError(); + + if (_ver!=Version::RNC1Old) + { + _rawCRC=packedData.readBE16(12); + _chunks=packedData.read8(17); + if (verify && !verified) + { + if (CRC16(_packedData,18,_packedSize,0)!=packedData.readBE16(14)) + throw VerificationError(); + } + } +} + +RNCDecompressor::~RNCDecompressor() +{ + // nothing needed +} + +const std::string &RNCDecompressor::getName() const noexcept +{ + static std::string names[3]={ + "RNC1: Rob Northen RNC1 Compressor (old)", + "RNC1: Rob Northen RNC1 Compressor ", + "RNC2: Rob Northen RNC2 Compressor"}; + return names[static_cast<uint32_t>(_ver)]; +} + +size_t RNCDecompressor::getPackedSize() const noexcept +{ + if (_ver==Version::RNC1Old) return _packedSize+12; + else return _packedSize+18; +} + +size_t RNCDecompressor::getRawSize() const noexcept +{ + return _rawSize; +} + +void RNCDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + if (rawData.size()<_rawSize) throw DecompressionError(); + + switch (_ver) + { + case Version::RNC1Old: + return RNC1DecompressOld(rawData,verify); + + case Version::RNC1New: + return RNC1DecompressNew(rawData,verify); + + case Version::RNC2: + return RNC2Decompress(rawData,verify); + + default: + throw DecompressionError(); + } +} + +void RNCDecompressor::RNC1DecompressOld(Buffer &rawData,bool verify) +{ + BackwardInputStream inputStream(_packedData,12,_packedSize+12); + MSBBitReader<BackwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + // the anchor-bit does not seem always to be at the correct place + { + uint8_t halfByte=readByte(); + for (uint32_t i=0;i<7;i++) + if (halfByte&(1<<i)) + { + bitReader.reset(halfByte>>(i+1),7-i); + break; + } + } + + BackwardOutputStream outputStream(rawData,0,_rawSize); + + HuffmanDecoder<uint8_t> litDecoder + { + HuffmanCode<uint8_t>{1,0b00,0}, + HuffmanCode<uint8_t>{2,0b10,1}, + HuffmanCode<uint8_t>{2,0b11,2} + }; + + HuffmanDecoder<uint8_t> lengthDecoder + { + HuffmanCode<uint8_t>{1,0b0000,0}, + HuffmanCode<uint8_t>{2,0b0010,1}, + HuffmanCode<uint8_t>{3,0b0110,2}, + HuffmanCode<uint8_t>{4,0b1110,3}, + HuffmanCode<uint8_t>{4,0b1111,4} + }; + + HuffmanDecoder<uint8_t> distanceDecoder + { + HuffmanCode<uint8_t>{1,0b00,0}, + HuffmanCode<uint8_t>{2,0b10,1}, + HuffmanCode<uint8_t>{2,0b11,2} + }; + + for (;;) + { + uint32_t litLength=litDecoder.decode(readBit); + + if (litLength==2) + { + static const uint32_t litBitLengths[4]={2,2,3,10}; + static const uint32_t litAdditions[4]={2,5,8,15}; + for (uint32_t i=0;i<4;i++) + { + litLength=readBits(litBitLengths[i]); + if (litLength!=(1U<<litBitLengths[i])-1U || i==3) + { + litLength+=litAdditions[i]; + break; + } + } + } + + for (uint32_t i=0;i<litLength;i++) outputStream.writeByte(readByte()); + + // the only way to successfully end the loop! + if (outputStream.eof()) break; + + uint32_t count; + { + uint32_t lengthIndex=lengthDecoder.decode(readBit); + static const uint32_t lengthBitLengths[5]={0,0,1,2,10}; + static const uint32_t lengthAdditions[5]={2,3,4,6,10}; + count=readBits(lengthBitLengths[lengthIndex])+lengthAdditions[lengthIndex]; + } + + uint32_t distance; + if (count!=2) + { + uint32_t distanceIndex=distanceDecoder.decode(readBit); + static const uint32_t distanceBitLengths[3]={8,5,12}; + static const uint32_t distanceAdditions[3]={32,0,288}; + distance=readBits(distanceBitLengths[distanceIndex])+distanceAdditions[distanceIndex]; + } else { + if (!readBit()) + { + distance=readBits(6); + } else { + distance=readBits(9)+64; + } + } + + outputStream.copy((distance)?distance+count-1:1,count); + } +} + +void RNCDecompressor::RNC1DecompressNew(Buffer &rawData,bool verify) +{ + ForwardInputStream inputStream(_packedData,18,_packedSize+18); + LSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits16Limit(count); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,_rawSize); + + typedef HuffmanDecoder<uint32_t> RNC1HuffmanDecoder; + + // helpers + auto readHuffmanTable=[&](RNC1HuffmanDecoder &dec) + { + uint32_t length=readBits(5); + if (!length) return; + uint32_t maxDepth=0; + uint8_t lengthTable[31]; + for (uint32_t i=0;i<length;i++) + { + lengthTable[i]=readBits(4); + if (lengthTable[i]>maxDepth) maxDepth=lengthTable[i]; + } + + dec.createOrderlyHuffmanTable(lengthTable,length); + }; + + auto huffmanDecode=[&](const RNC1HuffmanDecoder &dec)->int32_t + { + // this is kind of non-specced + uint32_t ret=dec.decode([&]()->uint32_t{return readBits(1);}); + if (ret>=2) + ret=(1<<(ret-1))|readBits(ret-1); + return ret; + }; + + auto processLiterals=[&](const RNC1HuffmanDecoder &dec) + { + uint32_t litLength=huffmanDecode(dec); + for (uint32_t i=0;i<litLength;i++) outputStream.writeByte(readByte()); + }; + + readBits(2); +#ifdef ALLOW_MISSING_CHUNKS + while (!outputStream.eof()) +#else + for (uint8_t chunks=0;chunks<_chunks;chunks++) +#endif + { + RNC1HuffmanDecoder litDecoder,distanceDecoder,lengthDecoder; + readHuffmanTable(litDecoder); + readHuffmanTable(distanceDecoder); + readHuffmanTable(lengthDecoder); + uint32_t count=readBits(16); + + for (uint32_t sub=1;sub<count;sub++) + { + processLiterals(litDecoder); + uint32_t distance=huffmanDecode(distanceDecoder); + uint32_t count=huffmanDecode(lengthDecoder); + distance++; + count+=2; + outputStream.copy(distance,count); + } + processLiterals(litDecoder); + } + + if (!outputStream.eof()) throw DecompressionError(); + if (verify && CRC16(rawData,0,_rawSize,0)!=_rawCRC) throw VerificationError(); +} + +void RNCDecompressor::RNC2Decompress(Buffer &rawData,bool verify) +{ + ForwardInputStream inputStream(_packedData,18,_packedSize+18); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,_rawSize); + + // Huffman decoding + enum class Cmd + { + LIT=0, // 0, Literal + MOV, // 10, Move bytes + length + distance, Get bytes if length=9 + 4bits + MV2, // 110, Move 2 bytes + MV3, // 1110, Move 3 bytes + CND // 1111, Conditional copy, or EOF + + }; + + HuffmanDecoder<Cmd> cmdDecoder + { + HuffmanCode<Cmd>{1,0b0000,Cmd::LIT}, + HuffmanCode<Cmd>{2,0b0010,Cmd::MOV}, + HuffmanCode<Cmd>{3,0b0110,Cmd::MV2}, + HuffmanCode<Cmd>{4,0b1110,Cmd::MV3}, + HuffmanCode<Cmd>{4,0b1111,Cmd::CND} + }; + + /* length of 9 is a marker for literals */ + HuffmanDecoder<uint8_t> lengthDecoder + { + HuffmanCode<uint8_t>{2,0b000,4}, + HuffmanCode<uint8_t>{2,0b010,5}, + HuffmanCode<uint8_t>{3,0b010,6}, + HuffmanCode<uint8_t>{3,0b011,7}, + HuffmanCode<uint8_t>{3,0b110,8}, + HuffmanCode<uint8_t>{3,0b111,9} + }; + + HuffmanDecoder<int8_t> distanceDecoder + { + HuffmanCode<int8_t>{1,0b000000,0}, + HuffmanCode<int8_t>{3,0b000110,1}, + HuffmanCode<int8_t>{4,0b001000,2}, + HuffmanCode<int8_t>{4,0b001001,3}, + HuffmanCode<int8_t>{5,0b010101,4}, + HuffmanCode<int8_t>{5,0b010111,5}, + HuffmanCode<int8_t>{5,0b011101,6}, + HuffmanCode<int8_t>{5,0b011111,7}, + HuffmanCode<int8_t>{6,0b101000,8}, + HuffmanCode<int8_t>{6,0b101001,9}, + HuffmanCode<int8_t>{6,0b101100,10}, + HuffmanCode<int8_t>{6,0b101101,11}, + HuffmanCode<int8_t>{6,0b111000,12}, + HuffmanCode<int8_t>{6,0b111001,13}, + HuffmanCode<int8_t>{6,0b111100,14}, + HuffmanCode<int8_t>{6,0b111101,15} + }; + + + // helpers + auto readDistance=[&]()->uint32_t + { + int8_t distMult=distanceDecoder.decode(readBit); + if (distMult<0) throw DecompressionError(); + uint8_t distByte=readByte(); + return (uint32_t(distByte)|(uint32_t(distMult)<<8))+1; + }; + + auto moveBytes=[&](uint32_t distance,uint32_t count)->void + { + if (!count) throw DecompressionError(); + outputStream.copy(distance,count); + }; + + readBit(); + readBit(); + uint8_t foundChunks=0; + bool done=false; + while (!done && foundChunks<_chunks) + { + Cmd cmd=cmdDecoder.decode(readBit); + switch (cmd) { + case Cmd::LIT: + outputStream.writeByte(readByte()); + break; + + case Cmd::MOV: + { + uint8_t count=lengthDecoder.decode(readBit); + if (count!=9) + moveBytes(readDistance(),count); + else { + uint32_t rep=0; + for (uint32_t i=0;i<4;i++) + rep=(rep<<1)|readBit(); + rep=(rep+3)*4; + for (uint32_t i=0;i<rep;i++) + outputStream.writeByte(readByte()); + } + } + break; + + case Cmd::MV2: + moveBytes(uint32_t(readByte())+1,2); + break; + + case Cmd::MV3: + moveBytes(readDistance(),3); + break; + + case Cmd::CND: + { + uint8_t count=readByte(); + if (count) + moveBytes(readDistance(),uint32_t(count+8)); + else { + foundChunks++; + done=!readBit(); + } + + } + break; + } + } + + if (!outputStream.eof() || _chunks!=foundChunks) throw DecompressionError(); + if (verify && CRC16(rawData,0,_rawSize,0)!=_rawCRC) throw VerificationError(); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/RNCDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RNCDecompressor.hpp new file mode 100644 index 00000000..b67820c7 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RNCDecompressor.hpp @@ -0,0 +1,51 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef RNCDECOMPRESSOR_HPP +#define RNCDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class RNCDecompressor : public Decompressor +{ +public: + RNCDecompressor(const Buffer &packedData,bool verify); + + virtual ~RNCDecompressor(); + + virtual const std::string &getName() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + virtual size_t getRawSize() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + + static bool detectHeader(uint32_t hdr) noexcept; + + static std::shared_ptr<Decompressor> create(const Buffer &packedData,bool exactSizeKnown,bool verify); + +private: + enum class Version + { + RNC1Old=0, + RNC1New, + RNC2 + }; + + void RNC1DecompressOld(Buffer &rawData,bool verify); + void RNC1DecompressNew(Buffer &rawData,bool verify); + void RNC2Decompress(Buffer &rawData,bool verify); + + const Buffer &_packedData; + + uint32_t _rawSize=0; + uint32_t _packedSize=0; + uint16_t _rawCRC=0; + uint8_t _chunks=0; + Version _ver; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/RangeDecoder.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RangeDecoder.cpp new file mode 100644 index 00000000..061c4edc --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RangeDecoder.cpp @@ -0,0 +1,65 @@ +/* Copyright (C) Teemu Suutari */ + +#include "RangeDecoder.hpp" + + +namespace ancient::internal +{ + +RangeDecoder::BitReader::BitReader() +{ + // nothing needed +} + +RangeDecoder::BitReader::~BitReader() +{ + // nothing needed +} + +RangeDecoder::RangeDecoder(BitReader &bitReader,uint16_t initialValue) : + _bitReader(bitReader), + _stream(initialValue) +{ + // nothing needed +} + +RangeDecoder::~RangeDecoder() +{ + // nothing needed +} + +uint16_t RangeDecoder::decode(uint16_t length) +{ + return ((uint32_t(_stream-_low)+1)*length-1)/(uint32_t(_high-_low)+1); +} + +void RangeDecoder::scale(uint16_t newLow,uint16_t newHigh,uint16_t newRange) +{ + uint32_t range=uint32_t(_high-_low)+1; + _high=(range*newHigh)/newRange+_low-1; + _low=(range*newLow)/newRange+_low; + + auto doubleContext=[&](uint16_t decr) + { + _low-=decr; + _high-=decr; + _stream-=decr; + _low<<=1; + _high=(_high<<1)|1U; + _stream=(_stream<<1)|_bitReader.readBit(); + }; + + for (;;) + { + if (_high<0x8000U) + { + doubleContext(0U); + } else if (_low>=0x8000U) { + doubleContext(0x8000U); + } else if (_low>=0x4000U && _high<0xc000U) { + doubleContext(0x4000U); + } else break; + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/RangeDecoder.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RangeDecoder.hpp new file mode 100644 index 00000000..a68512cf --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/RangeDecoder.hpp @@ -0,0 +1,40 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef RANGEDECODER_HPP +#define RANGEDECODER_HPP + +#include <cstdint> + +namespace ancient::internal +{ + +// used by too many compressors... +class RangeDecoder +{ +public: + class BitReader + { + public: + BitReader(); + virtual ~BitReader(); + + virtual uint32_t readBit()=0; + }; + + RangeDecoder(BitReader &bitReader,uint16_t initialValue); + ~RangeDecoder(); + + uint16_t decode(uint16_t length); + void scale(uint16_t newLow,uint16_t newHigh,uint16_t newRange); + +private: + BitReader &_bitReader; + + uint16_t _low=0; + uint16_t _high=0xffffU; + uint16_t _stream; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SDHCDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SDHCDecompressor.cpp new file mode 100644 index 00000000..4d10cdbc --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SDHCDecompressor.cpp @@ -0,0 +1,126 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstring> + +#include "common/SubBuffer.hpp" +#include "SDHCDecompressor.hpp" +#include "XPKMain.hpp" +#include "DLTADecode.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool SDHCDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("SDHC"); +} + +std::shared_ptr<XPKDecompressor> SDHCDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<SDHCDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +SDHCDecompressor::SDHCDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr) || _packedData.size()<2) + throw Decompressor::InvalidFormatError(); + _mode=_packedData.readBE16(0); + if (verify && (_mode&0x8000U)) + { + ConstSubBuffer src(_packedData,2,_packedData.size()-2); + XPKMain main(src,_recursionLevel+1,true); + } +} + +SDHCDecompressor::~SDHCDecompressor() +{ + // nothing needed +} + +const std::string &SDHCDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-SDHC: Sample delta huffman compressor"; + return name; +} + +void SDHCDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ConstSubBuffer src(_packedData,2,_packedData.size()-2); + if (_mode&0x8000U) + { + XPKMain main(src,_recursionLevel+1,false); + main.decompress(rawData,verify); + } else { + if (src.size()!=rawData.size()) throw Decompressor::DecompressionError(); + std::memcpy(rawData.data(),src.data(),src.size()); + } + + size_t length=rawData.size()&~3U; + + auto deltaDecodeMono=[&]() + { + uint8_t *buf=rawData.data(); + + uint16_t ctr=0; + for (size_t i=0;i<length;i+=2) + { + uint16_t tmp; + tmp=(uint16_t(buf[i])<<8)|uint16_t(buf[i+1]); + ctr+=tmp; + buf[i]=ctr>>8; + buf[i+1]=ctr&0xff; + } + }; + + auto deltaDecodeStereo=[&]() + { + uint8_t *buf=rawData.data(); + + uint16_t ctr1=0,ctr2=0; + for (size_t i=0;i<length;i+=4) + { + uint16_t tmp; + tmp=(uint16_t(buf[i])<<8)|uint16_t(buf[i+1]); + ctr1+=tmp; + tmp=(uint16_t(buf[i+2])<<8)|uint16_t(buf[i+3]); + ctr2+=tmp; + buf[i]=ctr1>>8; + buf[i+1]=ctr1&0xff; + buf[i+2]=ctr2>>8; + buf[i+3]=ctr2&0xff; + } + }; + + switch (_mode&15) + { + case 1: + DLTADecode::decode(rawData,rawData,0,length); + // intentional fall through + case 0: + DLTADecode::decode(rawData,rawData,0,length); + break; + + case 3: + deltaDecodeMono(); + // intentional fall through + case 2: + deltaDecodeMono(); + break; + + case 11: + deltaDecodeStereo(); + // intentional fall through + case 10: + deltaDecodeStereo(); + break; + + default: + throw Decompressor::DecompressionError(); + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SDHCDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SDHCDecompressor.hpp new file mode 100644 index 00000000..d64c840c --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SDHCDecompressor.hpp @@ -0,0 +1,33 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef SDHCDECOMPRESSOR_HPP +#define SDHCDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class SDHCDecompressor : public XPKDecompressor +{ +public: + SDHCDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~SDHCDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + uint16_t _mode=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SHR3Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SHR3Decompressor.cpp new file mode 100644 index 00000000..1ca4d66c --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SHR3Decompressor.cpp @@ -0,0 +1,326 @@ +/* Copyright (C) Teemu Suutari */ + +#include "SHR3Decompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +SHR3Decompressor::SHR3State::SHR3State() noexcept +{ + for (uint32_t i=0;i<999;i++) ar[i]=0; +} + +SHR3Decompressor::SHR3State::~SHR3State() +{ + // nothing needed +} + +bool SHR3Decompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("SHR3"); +} + +std::shared_ptr<XPKDecompressor> SHR3Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<SHR3Decompressor>(hdr,recursionLevel,packedData,state,verify); +} + +SHR3Decompressor::SHR3Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData), + _state(state) +{ + if (!detectHeaderXPK(hdr) || _packedData.size()<6) throw Decompressor::InvalidFormatError(); + _ver=_packedData.read8(0); + if (!_ver || _ver>2) throw Decompressor::InvalidFormatError(); + + if (!_state) + { + if (_ver==2) throw Decompressor::InvalidFormatError(); + _state.reset(new SHR3State()); + } +} + +SHR3Decompressor::~SHR3Decompressor() +{ + // nothing needed +} + +const std::string &SHR3Decompressor::getSubName() const noexcept +{ + static std::string name="XPK-SHR3: LZ-compressor with arithmetic encoding"; + return name; +} + +void SHR3Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,1,_packedData.size()); + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + // This follows quite closely Choloks pascal reference + uint32_t ar[999]; + + auto resum=[&]() + { + for (uint32_t i=498;i;i--) + ar[i]=ar[i*2]+ar[i*2+1]; + }; + + auto init=[&]() + { + for (uint32_t i=0;i<499;i++) ar[i]=0; + for (uint32_t i=0;i<256;i++) ar[i+499]=(i<32||i>126)?1:3; + for (uint32_t i=256+499;i<999;i++) ar[i]=0; + resum(); + }; + + auto update=[&](uint32_t updateIndex,uint32_t increment) + { + if (updateIndex>=499) return; + updateIndex+=499; + while (updateIndex) + { + ar[updateIndex]+=increment; + updateIndex>>=1; + } + if (ar[1]>=0x2000) + { + for (uint32_t i=499;i<998;i++) + if (ar[i]) ar[i]=(ar[i]>>1)+1; + resum(); + } + }; + + auto scale=[&](uint32_t b,uint32_t mult)->uint32_t + { + if (!b) throw Decompressor::DecompressionError(); + uint32_t tmp=(0x10000U/b); + uint32_t tmp2=((0x10000U%b)<<16)/b; + return ((mult&0xffffU)*tmp>>16)+((mult>>16)*tmp2>>16)+(mult>>16)*tmp; + }; + + uint32_t vlen=0,vnext=0; + + auto upgrade=[&]() + { + if (vnext>=65532) + { + vnext=~0U; + } else if (!vlen) { + vnext=1; + } else { + uint32_t vvalue=vnext-1; + if (vvalue<48) update(vvalue+256,1); + uint32_t bits=0,compare=4; + while (vvalue>=compare) + { + vvalue-=compare; + compare<<=1; + bits++; + } + if (bits>=14) + { + vnext=~0U; + } else { + if (!vvalue) + { + if (bits<7) + { + for (uint32_t i=304;i<=307;i++) + update((bits<<2)+i,1); + } + if (bits<13) + { + for (uint32_t i=332;i<=333;i++) + update((bits<<1)+i,1); + } + static const uint32_t updates1[6]={358,359,386,387,414,415}; + static const uint32_t updates2[4]={442,456,470,484}; + for (auto it : updates1) + update((bits<<1)+it,1); + for (auto it : updates2) + update(bits+it,1); + } + if (vnext<49) + { + vnext++; + } else if (vnext==49) { + vnext=61; + } else { + vnext=(vnext<<1)+3; + } + } + } + }; + + uint32_t stream=0,shift=0; + + auto refillStream=[&]() + { + while (shift<0x100'0000) + { + stream=(stream<<8)|uint32_t(readByte()); + shift<<=8; + } + }; + + auto getSymbol=[&]()->uint32_t + { + if (!(shift>>16)) throw Decompressor::DecompressionError(); + uint32_t vvalue=(stream/(shift>>16))&0xffff; + uint32_t threshold=(ar[1]*vvalue)>>16; + uint32_t arIndex=1; + uint32_t result=0; + do { + arIndex<<=1; + uint32_t tmp=ar[arIndex]+result; + if (threshold>=tmp) + { + result=tmp; + arIndex++; + } + } while (arIndex<499); + uint32_t rawValue=scale(ar[1],shift); + uint32_t newValue=rawValue*result; + if (newValue>stream) + { + while (newValue>stream) + { + if (--arIndex<499) arIndex+=499; + result-=ar[arIndex]; + newValue=rawValue*result; + } + } else { + result+=ar[arIndex]; + while (result<ar[1]) + { + uint32_t compare=rawValue*result; + if (stream<compare) break; + if (++arIndex>=998) arIndex-=499; + result+=ar[arIndex]; + newValue=compare; + } + } + stream-=newValue; + shift=rawValue*ar[arIndex]; + uint32_t addition=(ar[1]>>10)+3; + arIndex-=499; + update(arIndex,addition); + refillStream(); + return arIndex; + }; + + auto getCode=[&](uint32_t size)->uint32_t + { + uint32_t ret=0; + while (size--) + { + ret<<=1; + shift>>=1; + if (stream>=shift) + { + ret++; + stream-=shift; + } + refillStream(); + } + return ret; + }; + + if (_ver==1) + { + init(); + update(498,1); + + shift=0x8000'0000U; + } else { + SHR3State *state=static_cast<SHR3State*>(_state.get()); + vlen=state->vlen; + vnext=state->vnext; + shift=state->shift; + for (uint32_t i=0;i<999;i++) ar[i]=state->ar[i]; + } + + { + const uint8_t *buf=inputStream.consume(4); + stream=(uint32_t(buf[0])<<24)|(uint32_t(buf[1])<<16)| + (uint32_t(buf[2])<<8)|uint32_t(buf[3]); + } + + while (!outputStream.eof()) + { + while (vlen>=vnext) upgrade(); + uint32_t code=getSymbol(); + if (code<256) + { + outputStream.writeByte(code); + vlen++; + } else { + auto distanceAddition=[](uint32_t i)->uint32_t + { + return ((1<<(i+2))-1)&~0x3U; + }; + + uint32_t count,distance; + if (code<304) + { + count=2; + distance=code-255; + } else if (code<332) { + uint32_t tmp=code-304; + uint32_t extra=getCode(tmp>>2); + distance=((extra<<2)|(tmp&3))+distanceAddition(tmp>>2)+1; + count=3; + } else if (code<358) { + uint32_t tmp=code-332; + uint32_t extra=getCode((tmp>>1)+1); + distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; + count=4; + } else if (code<386) { + uint32_t tmp=code-358; + uint32_t extra=getCode((tmp>>1)+1); + distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; + count=5; + } else if (code<414) { + uint32_t tmp=code-386; + uint32_t extra=getCode((tmp>>1)+1); + distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; + count=6; + } else if (code<442) { + uint32_t tmp=code-414; + uint32_t extra=getCode((tmp>>1)+1); + distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; + count=7; + } else if (code<498) { + uint32_t tmp=code-442; + uint32_t d=tmp/14; + uint32_t m=tmp%14; + count=getCode(d+2)+distanceAddition(d)+8; + distance=getCode(m+2)+distanceAddition(m)+1; + } else { + count=getCode(16); + distance=getCode(16); + } + vlen+=count; + if (!count) throw Decompressor::DecompressionError(); + outputStream.copy(distance,count,previousData); + } + } + + SHR3State *state=static_cast<SHR3State*>(_state.get()); + state->vlen=vlen; + state->vnext=vnext; + state->shift=shift; + for (uint32_t i=0;i<999;i++) state->ar[i]=ar[i]; +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SHR3Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SHR3Decompressor.hpp new file mode 100644 index 00000000..2ea24191 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SHR3Decompressor.hpp @@ -0,0 +1,48 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef SHR3DECOMPRESSOR_HPP +#define SHR3DECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class SHR3Decompressor : public XPKDecompressor +{ +private: + class SHR3State : public XPKDecompressor::State + { + public: + SHR3State() noexcept; + virtual ~SHR3State(); + + uint32_t vlen=0; + uint32_t vnext=0; + uint32_t shift=0; + uint32_t ar[999]; + }; + +public: + SHR3Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~SHR3Decompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _ver=0; + + std::shared_ptr<XPKDecompressor::State> &_state; // reference!!! +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SHRIDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SHRIDecompressor.cpp new file mode 100644 index 00000000..92723cb6 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SHRIDecompressor.cpp @@ -0,0 +1,338 @@ +/* Copyright (C) Teemu Suutari */ + +#include "SHRIDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +SHRIDecompressor::SHRIState::SHRIState() noexcept +{ + for (uint32_t i=0;i<999;i++) ar[i]=0; +} + +SHRIDecompressor::SHRIState::~SHRIState() +{ + // nothing needed +} + +bool SHRIDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("SHRI"); +} + +std::shared_ptr<XPKDecompressor> SHRIDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<SHRIDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +SHRIDecompressor::SHRIDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData), + _state(state) +{ + if (!detectHeaderXPK(hdr) || _packedData.size()<6) throw Decompressor::InvalidFormatError(); + _ver=_packedData.read8(0); + if (!_ver || _ver>2) throw Decompressor::InvalidFormatError(); + + // second byte defines something that does not seem to be terribly important... + uint8_t tmp=_packedData.read8(2); + if (tmp<0x80) + { + _rawSize=_packedData.readBE16(2); + _startOffset=4; + } else { + _rawSize=~_packedData.readBE32(2)+1U; + _startOffset=6; + } + + if (!_state) + { + if (_ver==2) throw Decompressor::InvalidFormatError(); + _state.reset(new SHRIState()); + } +} + +SHRIDecompressor::~SHRIDecompressor() +{ + // nothing needed +} + +const std::string &SHRIDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-SHRI: LZ-compressor with arithmetic encoding"; + return name; +} + +void SHRIDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); + + ForwardInputStream inputStream(_packedData,_startOffset,_packedData.size()); + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + // This follows quite closely Choloks pascal reference + uint32_t ar[999]; + + auto resum=[&]() + { + for (uint32_t i=498;i;i--) + ar[i]=ar[i*2]+ar[i*2+1]; + }; + + auto init=[&]() + { + for (uint32_t i=0;i<499;i++) ar[i]=0; + for (uint32_t i=0;i<256;i++) ar[i+499]=(i<32||i>126)?1:3; + for (uint32_t i=256+499;i<999;i++) ar[i]=0; + resum(); + }; + + auto update=[&](uint32_t updateIndex,uint32_t increment) + { + if (updateIndex>=499) return; + updateIndex+=499; + while (updateIndex) + { + ar[updateIndex]+=increment; + updateIndex>>=1; + } + if (ar[1]>=0x2000) + { + for (uint32_t i=499;i<998;i++) + if (ar[i]) ar[i]=(ar[i]>>1)+1; + resum(); + } + }; + + auto scale=[&](uint32_t a,uint32_t b,uint32_t mult)->uint32_t + { + if (!b) throw Decompressor::DecompressionError(); + uint32_t tmp=(a<<16)/b; + uint32_t tmp2=(((a<<16)%b)<<16)/b; + return ((mult&0xffffU)*tmp>>16)+((mult>>16)*tmp2>>16)+(mult>>16)*tmp; + }; + + uint32_t vlen=0,vnext=0; + + auto upgrade=[&]() + { + if (vnext>=65532) + { + vnext=~0U; + } else if (!vlen) { + vnext=1; + } else { + uint32_t vvalue=vnext-1; + if (vvalue<48) update(vvalue+256,1); + uint32_t bits=0,compare=4; + while (vvalue>=compare) + { + vvalue-=compare; + compare<<=1; + bits++; + } + if (bits>=14) + { + vnext=~0U; + } else { + if (!vvalue) + { + if (bits<7) + { + for (uint32_t i=304;i<=307;i++) + update((bits<<2)+i,1); + } + if (bits<13) + { + for (uint32_t i=332;i<=333;i++) + update((bits<<1)+i,1); + } + static const uint32_t updates1[6]={358,359,386,387,414,415}; + static const uint32_t updates2[4]={442,456,470,484}; + for (auto it : updates1) + update((bits<<1)+it,1); + for (auto it : updates2) + update(bits+it,1); + } + if (vnext<49) + { + vnext++; + } else if (vnext==49) { + vnext=61; + } else { + vnext=(vnext<<1)+3; + } + } + } + }; + + uint32_t stream=0,shift=0; + + auto refillStream=[&]() + { + while (shift<0x100'0000) + { + stream=(stream<<8)|uint32_t(readByte()); + shift<<=8; + } + }; + + auto getSymbol=[&]()->uint32_t + { + if (!(shift>>16)) throw Decompressor::DecompressionError(); + uint32_t vvalue=(stream/(shift>>16))&0xffff; + uint32_t threshold=(ar[1]*vvalue)>>16; + uint32_t arIndex=1; + uint32_t result=0; + do { + arIndex<<=1; + uint32_t tmp=ar[arIndex]+result; + if (threshold>=tmp) + { + result=tmp; + arIndex++; + } + } while (arIndex<499); + uint32_t newValue=scale(result,ar[1],shift); + if (newValue>stream) + { + while (newValue>stream) + { + if (--arIndex<499) arIndex+=499; + result-=ar[arIndex]; + newValue=scale(result,ar[1],shift); + } + } else { + result+=ar[arIndex]; + while (result<ar[1]) + { + uint32_t compare=scale(result,ar[1],shift); + if (stream<compare) break; + if (++arIndex>=998) arIndex-=499; + result+=ar[arIndex]; + newValue=compare; + } + } + stream-=newValue; + shift=scale(ar[arIndex],ar[1],shift); + uint32_t addition=(ar[1]>>10)+3; + arIndex-=499; + update(arIndex,addition); + refillStream(); + return arIndex; + }; + + auto getCode=[&](uint32_t size)->uint32_t + { + uint32_t ret=0; + while (size--) + { + ret<<=1; + shift>>=1; + if (stream>=shift) + { + ret++; + stream-=shift; + } + refillStream(); + } + return ret; + }; + + if (_ver==1) + { + init(); + update(498,1); + + shift=0x8000'0000U; + } else { + SHRIState *state=static_cast<SHRIState*>(_state.get()); + vlen=state->vlen; + vnext=state->vnext; + shift=state->shift; + for (uint32_t i=0;i<999;i++) ar[i]=state->ar[i]; + } + + { + const uint8_t *buf=inputStream.consume(4); + stream=(uint32_t(buf[0])<<24)|(uint32_t(buf[1])<<16)| + (uint32_t(buf[2])<<8)|uint32_t(buf[3]); + } + + while (!outputStream.eof()) + { + while (vlen>=vnext) upgrade(); + uint32_t code=getSymbol(); + if (code<256) + { + outputStream.writeByte(code); + vlen++; + } else { + auto distanceAddition=[](uint32_t i)->uint32_t + { + return ((1<<(i+2))-1)&~0x3U; + }; + + uint32_t count,distance; + if (code<304) + { + count=2; + distance=code-255; + } else if (code<332) { + uint32_t tmp=code-304; + uint32_t extra=getCode(tmp>>2); + distance=((extra<<2)|(tmp&3))+distanceAddition(tmp>>2)+1; + count=3; + } else if (code<358) { + uint32_t tmp=code-332; + uint32_t extra=getCode((tmp>>1)+1); + distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; + count=4; + } else if (code<386) { + uint32_t tmp=code-358; + uint32_t extra=getCode((tmp>>1)+1); + distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; + count=5; + } else if (code<414) { + uint32_t tmp=code-386; + uint32_t extra=getCode((tmp>>1)+1); + distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; + count=6; + } else if (code<442) { + uint32_t tmp=code-414; + uint32_t extra=getCode((tmp>>1)+1); + distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; + count=7; + } else if (code<498) { + uint32_t tmp=code-442; + uint32_t d=tmp/14; + uint32_t m=tmp%14; + count=getCode(d+2)+distanceAddition(d)+8; + distance=getCode(m+2)+distanceAddition(m)+1; + } else { + count=getCode(16); + distance=getCode(16); + } + vlen+=count; + if (!count) throw Decompressor::DecompressionError(); + outputStream.copy(distance,count,previousData); + } + } + + SHRIState *state=static_cast<SHRIState*>(_state.get()); + state->vlen=vlen; + state->vnext=vnext; + state->shift=shift; + for (uint32_t i=0;i<999;i++) state->ar[i]=ar[i]; +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SHRIDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SHRIDecompressor.hpp new file mode 100644 index 00000000..549fa1e0 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SHRIDecompressor.hpp @@ -0,0 +1,50 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef SHRIDECOMPRESSOR_HPP +#define SHRIDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class SHRIDecompressor : public XPKDecompressor +{ +private: + class SHRIState : public XPKDecompressor::State + { + public: + SHRIState() noexcept; + virtual ~SHRIState(); + + uint32_t vlen=0; + uint32_t vnext=0; + uint32_t shift=0; + uint32_t ar[999]; + }; + +public: + SHRIDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~SHRIDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _ver=0; + size_t _startOffset=0; + size_t _rawSize=0; + + std::shared_ptr<XPKDecompressor::State> &_state; // reference!!! +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SLZ3Decompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SLZ3Decompressor.cpp new file mode 100644 index 00000000..199a8689 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SLZ3Decompressor.cpp @@ -0,0 +1,71 @@ +/* Copyright (C) Teemu Suutari */ + +#include "SLZ3Decompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool SLZ3Decompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("SLZ3"); +} + +std::shared_ptr<XPKDecompressor> SLZ3Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<SLZ3Decompressor>(hdr,recursionLevel,packedData,state,verify); +} + +SLZ3Decompressor::SLZ3Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +SLZ3Decompressor::~SLZ3Decompressor() +{ + // nothing needed +} + +const std::string &SLZ3Decompressor::getSubName() const noexcept +{ + static std::string name="XPK-SLZ3: SLZ3 CyberYAFA compressor"; + return name; +} + +void SLZ3Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + while (!outputStream.eof()) + { + if (!readBit()) + { + outputStream.writeByte(readByte()); + } else { + uint8_t tmp=readByte(); + if (!tmp) throw Decompressor::DecompressionError(); + uint32_t distance=uint32_t(tmp&0xf0)<<4; + distance|=uint32_t(readByte()); + uint32_t count=uint32_t(tmp&0xf)+2; + outputStream.copy(distance,count); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SLZ3Decompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SLZ3Decompressor.hpp new file mode 100644 index 00000000..56f62800 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SLZ3Decompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef SLZ3DECOMPRESSOR_HPP +#define SLZ3DECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class SLZ3Decompressor : public XPKDecompressor +{ +public: + SLZ3Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~SLZ3Decompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SMPLDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SMPLDecompressor.cpp new file mode 100644 index 00000000..95530147 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SMPLDecompressor.cpp @@ -0,0 +1,78 @@ +/* Copyright (C) Teemu Suutari */ + +#include "SMPLDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "HuffmanDecoder.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool SMPLDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("SMPL"); +} + +std::shared_ptr<XPKDecompressor> SMPLDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<SMPLDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +SMPLDecompressor::SMPLDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr) || packedData.size()<2) throw Decompressor::InvalidFormatError(); + + if (packedData.readBE16(0)!=1) throw Decompressor::InvalidFormatError(); +} + +SMPLDecompressor::~SMPLDecompressor() +{ + // nothing needed +} + +const std::string &SMPLDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-SMPL: Huffman compressor with delta encoding"; + return name; +} + +void SMPLDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,2,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + HuffmanDecoder<uint32_t> decoder; + + for (uint32_t i=0;i<256;i++) + { + uint32_t codeLength=readBits(4); + if (!codeLength) continue; + if (codeLength==15) codeLength=readBits(4)+15; + uint32_t code=readBits(codeLength); + decoder.insert(HuffmanCode<uint32_t>{codeLength,code,i}); + } + + uint8_t accum=0; + while (!outputStream.eof()) + { + uint32_t code=decoder.decode(readBit); + accum+=code; + outputStream.writeByte(accum); + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SMPLDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SMPLDecompressor.hpp new file mode 100644 index 00000000..e2153ef9 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SMPLDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef SMPLDECOMPRESSOR_HPP +#define SMPLDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class SMPLDecompressor : public XPKDecompressor +{ +public: + SMPLDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~SMPLDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SQSHDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SQSHDecompressor.cpp new file mode 100644 index 00000000..bcf6383f --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SQSHDecompressor.cpp @@ -0,0 +1,218 @@ +/* Copyright (C) Teemu Suutari */ + +#include "SQSHDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "HuffmanDecoder.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool SQSHDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("SQSH"); +} + +std::shared_ptr<XPKDecompressor> SQSHDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<SQSHDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +SQSHDecompressor::SQSHDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr) || packedData.size()<3) throw Decompressor::InvalidFormatError(); + _rawSize=packedData.readBE16(0); + if (!_rawSize) throw Decompressor::InvalidFormatError(); +} + +SQSHDecompressor::~SQSHDecompressor() +{ + // nothing needed +} + +const std::string &SQSHDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-SQSH: Compressor for sampled sounds"; + return name; +} + +void SQSHDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); + + ForwardInputStream inputStream(_packedData,2,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readSignedBits=[&](uint8_t bits)->int32_t + { + int32_t ret=readBits(bits); + if (ret&(1<<(bits-1))) + ret|=~0U<<bits; + return ret; + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,_rawSize); + + HuffmanDecoder<uint8_t> modDecoder + { + HuffmanCode<uint8_t>{1,0b0001,0}, + HuffmanCode<uint8_t>{2,0b0000,1}, + HuffmanCode<uint8_t>{3,0b0010,2}, + HuffmanCode<uint8_t>{4,0b0110,3}, + HuffmanCode<uint8_t>{4,0b0111,4} + }; + + HuffmanDecoder<uint8_t> lengthDecoder + { + HuffmanCode<uint8_t>{1,0b0000,0}, + HuffmanCode<uint8_t>{2,0b0010,1}, + HuffmanCode<uint8_t>{3,0b0110,2}, + HuffmanCode<uint8_t>{4,0b1110,3}, + HuffmanCode<uint8_t>{4,0b1111,4} + }; + + HuffmanDecoder<uint8_t> distanceDecoder + { + HuffmanCode<uint8_t>{1,0b01,0}, + HuffmanCode<uint8_t>{2,0b00,1}, + HuffmanCode<uint8_t>{2,0b01,2} + }; + + // first byte is special + uint8_t currentSample=readByte(); + outputStream.writeByte(currentSample); + + uint32_t accum1=0,accum2=0,prevBits=0; + + while (!outputStream.eof()) + { + uint8_t bits=0; + uint32_t count=0; + bool doRepeat=false; + + if (accum1>=8) + { + static const uint8_t bitLengthTable[7][8]={ + {2,3,4,5,6,7,8,0}, + {3,2,4,5,6,7,8,0}, + {4,3,5,2,6,7,8,0}, + {5,4,6,2,3,7,8,0}, + {6,5,7,2,3,4,8,0}, + {7,6,8,2,3,4,5,0}, + {8,7,6,2,3,4,5,0}}; + + auto handleCondCase=[&]() + { + if (bits==8) { + if (accum2<20) + { + count=1; + } else { + count=2; + accum2+=8; + } + } else { + count=5; + accum2+=8; + } + }; + + auto handleTable=[&](uint32_t newBits) + { + if (prevBits<2 || !newBits) throw Decompressor::DecompressionError(); + bits=bitLengthTable[prevBits-2][newBits-1]; + if (!bits) throw Decompressor::DecompressionError(); + handleCondCase(); + }; + + uint32_t mod=modDecoder.decode(readBit); + switch (mod) + { + case 0: + if (prevBits==8) + { + bits=8; + handleCondCase(); + } else { + bits=prevBits; + count=5; + accum2+=8; + } + break; + + case 1: + doRepeat=true; + break; + + case 2: + handleTable(2); + break; + + case 3: + handleTable(3); + break; + + case 4: + handleTable(readBits(2)+4); + break; + + default: + throw Decompressor::DecompressionError(); + } + } else { + if (readBit()) + { + doRepeat=true; + } else { + count=1; + bits=8; + } + } + + if (doRepeat) { + uint32_t lengthIndex=lengthDecoder.decode(readBit); + static const uint8_t lengthBits[5]={1,1,1,3,5}; + static const uint32_t lengthAdditions[5]={2,4,6,8,16}; + count=readBits(lengthBits[lengthIndex])+lengthAdditions[lengthIndex]; + if (count>=3) + { + if (accum1) accum1--; + if (count>3 && accum1) accum1--; + } + uint32_t distanceIndex=distanceDecoder.decode(readBit); + static const uint8_t distanceBits[3]={12,8,14}; + static const uint32_t distanceAdditions[3]={0x101,1,0x1101}; + uint32_t distance=readBits(distanceBits[distanceIndex])+distanceAdditions[distanceIndex]; + count=std::min(count,uint32_t(_rawSize-outputStream.getOffset())); + currentSample=outputStream.copy(distance,count); + } else { + count=std::min(count,uint32_t(_rawSize-outputStream.getOffset())); + for (uint32_t i=0;i<count;i++) + { + currentSample-=readSignedBits(bits); + outputStream.writeByte(currentSample); + } + if (accum1!=31) accum1++; + prevBits=bits; + } + + accum2-=accum2>>3; + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SQSHDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SQSHDecompressor.hpp new file mode 100644 index 00000000..066e0905 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SQSHDecompressor.hpp @@ -0,0 +1,33 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef SQSHDECOMPRESSOR_HPP +#define SQSHDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class SQSHDecompressor : public XPKDecompressor +{ +public: + SQSHDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~SQSHDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _rawSize=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SXSCDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SXSCDecompressor.cpp new file mode 100644 index 00000000..73967362 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SXSCDecompressor.cpp @@ -0,0 +1,829 @@ +/* Copyright (C) Teemu Suutari */ + +#include "SXSCDecompressor.hpp" + +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "DLTADecode.hpp" +#include "common/MemoryBuffer.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +SXSCDecompressor::SXSCReader::SXSCReader(ForwardInputStream &stream) : + _reader(stream) +{ + // nothing needed +} + +SXSCDecompressor::SXSCReader::~SXSCReader() +{ + // nothing needed +} + +uint32_t SXSCDecompressor::SXSCReader::readBit() +{ + return _reader.readBits8(1); +} + + +bool SXSCDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("SASC")||hdr==FourCC("SHSC"); +} + +std::shared_ptr<XPKDecompressor> SXSCDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<SXSCDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +SXSCDecompressor::SXSCDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData), + _isHSC(hdr==FourCC("SHSC")) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +SXSCDecompressor::~SXSCDecompressor() +{ + // nothing needed +} + +const std::string &SXSCDecompressor::getSubName() const noexcept +{ + static std::string nameASC="XPK-SASC: LZ-compressor with arithmetic and delta encoding"; + static std::string nameHSC="XPK-SHSC: Context modeling compressor"; + return _isHSC?nameHSC:nameASC; +} + +void SXSCDecompressor::decompressASC(Buffer &rawData,ForwardInputStream &inputStream) +{ + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + uint16_t bitReaderInitialValue; + bitReaderInitialValue=inputStream.readByte()<<8; + bitReaderInitialValue|=inputStream.readByte(); + SXSCReader bitReader(inputStream); + RangeDecoder arithDecoder(bitReader,bitReaderInitialValue); + + // decoder for literal, copy, end decision + // two thresholds -> 3 symbols, last symbol is break with size of 1 + uint16_t bitThreshold1[4]={40,40,40,40}; + uint16_t bitThreshold2[4]={40,40,40,40}; + uint32_t bitPos=0; + + // generics for the other decoder + + auto tableElements=[](auto &table)->uint16_t + { + return (sizeof(table)/sizeof(*table)+1)>>1; + }; + + auto initTable=[](auto &table,uint16_t initialValue) + { + constexpr uint32_t length=(sizeof(table)/sizeof(*table)+1)>>1; + for (uint32_t i=0;i<length;i++) table[i]=initialValue; + for (uint32_t j=0,i=length;i<length*2-1;i++,j+=2) + table[i]=table[j]+table[j+1]; + }; + + auto updateTable=[](auto &table,uint16_t max,uint16_t index,int16_t value) + { + constexpr uint32_t length=(sizeof(table)/sizeof(*table)+1)>>1; + for (uint32_t i=index;i<length*2-1;i=(i>>1)+length) + table[i]+=value; + if (table[length*2-2]>=max) + { + for (uint32_t i=0;i<length;i++) + if (table[i]>1) table[i]>>=1; + for (uint32_t j=0,i=length;i<length*2-1;i++,j+=2) + table[i]=table[j]+table[j+1]; + } + }; + + auto tableSize=[](auto &table)->uint16_t + { + constexpr uint32_t length=(sizeof(table)/sizeof(*table)+1)>>1; + return table[length*2-2]; + }; + + auto decodeSymbol=[](auto &table,uint16_t &value)->uint16_t + { + constexpr uint32_t length=(sizeof(table)/sizeof(*table)+1)>>1; + uint32_t threshold=0; + uint32_t i=length*2-4; + while (i>=length) + { + uint32_t child=(i-length)<<1; + if (value-threshold>=table[i]) + { + threshold+=table[i]; + child+=2; + } + i=child; + } + if (value-threshold>=table[i]) + { + threshold+=table[i]; + i++; + } + value=threshold; + return i; + }; + + // literal decoder + uint16_t litInitial[256*2-1]; + uint16_t litDynamic[256*2-1]; + uint16_t litThreshold=1; + + initTable(litInitial,1); + initTable(litDynamic,0); + + // distance / length decoder + uint16_t distanceCodes[16*2-1]; + uint16_t countInitial[64*2-1]; + uint16_t countDynamic[64*2-1]; + uint16_t countThreshold=8; + + initTable(distanceCodes,0); + initTable(countInitial,1); + initTable(countDynamic,0); + + updateTable(distanceCodes,6000,0,24); + uint32_t distanceIndex=0; + + auto twoStepArithDecoder=[&](auto &initialTable,auto &dynamicTable,uint16_t &threshold,uint16_t max,uint16_t step,uint16_t updateRange)->uint16_t + { + uint16_t value=arithDecoder.decode(tableSize(dynamicTable)+threshold); + uint16_t ret; + if (value<tableSize(dynamicTable)) + { + ret=decodeSymbol(dynamicTable,value); + arithDecoder.scale(value,value+dynamicTable[ret],tableSize(dynamicTable)+threshold); + } else { + arithDecoder.scale(tableSize(dynamicTable),tableSize(dynamicTable)+threshold,tableSize(dynamicTable)+threshold); + + value=arithDecoder.decode(tableSize(initialTable)); + ret=decodeSymbol(initialTable,value); + arithDecoder.scale(value,value+initialTable[ret],tableSize(initialTable)); + updateTable(initialTable,65535,ret,-initialTable[ret]); + if (tableSize(initialTable)) threshold+=step; + else threshold=0; + for (uint32_t i=ret>updateRange?ret-updateRange:0;i<std::min(uint16_t(ret+updateRange),uint16_t(tableElements(initialTable)-1U));i++) + if (initialTable[i]) updateTable(initialTable,max,i,1); + } + updateTable(dynamicTable,max,ret,step); + if (dynamicTable[ret]==step*3) threshold=threshold>step?threshold-step:1; + return ret; + }; + + for (;;) + { + uint16_t bitSize=bitThreshold1[bitPos]+bitThreshold2[bitPos]; + uint16_t bitValue=arithDecoder.decode(bitSize+1); + if (bitValue==bitSize) break; + bool bit=bitValue<bitThreshold1[bitPos]; + arithDecoder.scale(bit?0:bitThreshold1[bitPos],bit?bitThreshold1[bitPos]:bitSize,bitSize+1); + (bit?bitThreshold1:bitThreshold2)[bitPos]+=40; + if (bitSize>=6000) + { + if (!(bitThreshold1[bitPos]>>=1)) bitThreshold1[bitPos]=1; + if (!(bitThreshold2[bitPos]>>=1)) bitThreshold2[bitPos]=1; + } + bitPos=(bitPos<<1&2)|(bit?0:1); + if (bit) + { + // literal + outputStream.writeByte(uint8_t(twoStepArithDecoder(litInitial,litDynamic,litThreshold,1000,1,8))); + } else { + // copy + while (outputStream.getOffset()>(1ULL<<distanceIndex) && distanceIndex<15U) + updateTable(distanceCodes,6000,++distanceIndex,24); + uint16_t distanceValue=arithDecoder.decode(tableSize(distanceCodes)); + uint16_t distanceBits=decodeSymbol(distanceCodes,distanceValue); + arithDecoder.scale(distanceValue,distanceValue+distanceCodes[distanceBits],tableSize(distanceCodes)); + updateTable(distanceCodes,6000,distanceBits,24); + uint32_t distance=distanceBits; + if (distanceBits>=2) + { + uint16_t minRange=1<<(distanceBits-1); + uint16_t range=distanceIndex==distanceBits?static_cast<uint16_t>(std::min(outputStream.getOffset(),size_t(31200U)))-minRange:minRange; + distance=arithDecoder.decode(range); + arithDecoder.scale(distance,distance+1,range); + distance+=minRange; + } + distance++; + uint32_t count=twoStepArithDecoder(countInitial,countDynamic,countThreshold,6000,8,4); + if (count==15) + { + count=783; + } else if (count>=16) { + uint16_t value=arithDecoder.decode(16); + arithDecoder.scale(value,value+1,16); + count=((count-16)<<4)+value+15; + } + count+=3; + outputStream.copy(distance,count); + } + } + if (!outputStream.eof()) throw Decompressor::DecompressionError(); +} + +template<typename T,size_t length> +class CheckedArray +{ +public: + CheckedArray() : + _memory(length*sizeof(T)) + { + // nothing needed + } + + ~CheckedArray() + { + // nothing needed + } + + T &operator[](size_t i) + { + if (i>=length) throw Decompressor::DecompressionError(); + return _memory.cast<T>()[i]; + } + + const T &operator[](size_t i) const + { + if (i>=length) throw Decompressor::DecompressionError(); + return _memory.cast<T>()[i]; + } + +private: + MemoryBuffer _memory; +}; + +// The horror. It needs to follow exactly the original logic, even if that logic is not very good. +void SXSCDecompressor::decompressHSC(Buffer &rawData,ForwardInputStream &inputStream) +{ + struct Model + { + uint8_t context[4]; + + uint16_t hashPointer; + uint16_t expiryPrevious; + uint16_t expiryNext; + uint16_t frequencyTotal; + uint16_t escapeFrequency; + + uint8_t contextLength; + uint8_t characterCount; + uint8_t refreshCounter; + }; + + struct Frequency + { + uint16_t frequency; + uint16_t next; + uint8_t character; + }; + + struct HashItem + { + uint16_t data; + uint16_t random; + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + uint16_t bitReaderInitialValue; + bitReaderInitialValue=inputStream.readByte()<<8; + bitReaderInitialValue|=inputStream.readByte(); + SXSCReader bitReader(inputStream); + RangeDecoder arithDecoder(bitReader,bitReaderInitialValue); + + uint8_t maxContextLength=4; + int16_t dropCount=2500; + int16_t contextSearchLength=0; + + CheckedArray<Model,10000> models{}; + for (uint32_t i=0;i<10000;i++) + { + auto &m=models[i]; + + for (uint32_t j=0;j<4;j++) + m.context[j]=0; + + m.hashPointer=0; + m.expiryPrevious=i-1; + m.expiryNext=i+1; + m.frequencyTotal=0; + m.escapeFrequency=0; + + m.contextLength=0xffU; + m.characterCount=0; + m.refreshCounter=0; + } + + uint8_t currentContext[4]; + for (uint32_t i=0;i<4;i++) currentContext[i]=0; + uint16_t firstExpiry=0,lastExpiry=9999; + uint16_t freeBlockPointer=10000; + uint16_t releaseBlock=0; + + CheckedArray<Frequency,32760> frequencies{}; + for (uint32_t i=0;i<32760;i++) + { + auto &f=frequencies[i]; + f.frequency=0; + f.next=(i>=10000&&i<32759)?i+1:0xffffU; + f.character=0; + } + + CheckedArray<HashItem,0x4000> hashes{}; + for (uint32_t i=0,j=10;i<0x4000U;i++) + { + auto &h=hashes[i]; + h.data=0xffffU; + // constants used are 2147483647 % / 16807 + int32_t integerPart=j/127773U; + int32_t fractionalPart=j%127773U; + int32_t tmp=16807*fractionalPart-2836*integerPart; + j=tmp<0?tmp+0x7fff'ffff:tmp; + h.random=j&0x3fffU; + } + uint16_t hashStack[5]; + for (uint32_t i=0;i<5;i++) + hashStack[i]=0; + + bool characterMask[256]; + uint8_t characterMaskStack[256]; + for (uint32_t i=0;i<256;i++) + { + characterMask[i]=false; + characterMaskStack[i]=0; + } + int16_t characterMaskStackPointer=0; + + uint8_t initialEscapeChar[5]; + for (uint32_t i=0;i<5;i++) + initialEscapeChar[i]=i?15:16; + uint8_t escapeCharacterCounter=0; + + uint16_t stackPointer=0; + uint16_t contextPointer[5]; + uint16_t frequencyArrayIndex[5]; + for (uint32_t i=0;i<5;i++) + { + contextPointer[i]=0; + frequencyArrayIndex[i]=0; + } + + auto loopBreaker=[](uint16_t i) { + if (i>=0x8000U) throw Decompressor::DecompressionError(); + }; + + auto findNext=[&]()->uint16_t + { + for (int32_t i=contextSearchLength-1;i>=0;i--) + { + for (uint32_t lb=0,j=hashes[hashStack[i]].data;j!=0xffffU;j=models[j].hashPointer,loopBreaker(lb++)) + { + if (i==models[j].contextLength) + { + if ([&]()->bool + { + for (int32_t k=0;k<i;k++) + if (currentContext[k]!=models[j].context[k]) return false; + return true; + }()) { + contextSearchLength=i; + return j; + } + } + } + } + return 0xffffU; + }; + + for (;;) + { + for (uint32_t i=0;i<4;i++) + hashStack[i+1]=hashes[(currentContext[i]+hashStack[i])&0x3fffU].random; + stackPointer=0; + while (characterMaskStackPointer) + characterMask[characterMaskStack[--characterMaskStackPointer]]=false; + contextSearchLength=5; + uint16_t index=findNext(); + + uint8_t minLength=index!=0xffffU?models[index].contextLength+1:0; + uint16_t ch; + for (;;index=findNext()) + { + if (index==0xffffU) + { + uint16_t size=257-characterMaskStackPointer; + uint16_t value=arithDecoder.decode(size); + // logic from original, could be improved a lot + // however, lets not optimize unless it is a problem... + uint16_t i=0; + for (ch=0;ch<256;ch++) + { + if (characterMask[ch]) continue; + if (i>=value) break; + i++; + } + arithDecoder.scale(i,i+1,size); + break; + } + + // madness!!! + auto getEscFrequency=[&](uint16_t value,uint16_t i)->uint16_t + { + auto &model=models[i]; + if (model.frequencyTotal==1) + return initialEscapeChar[model.contextLength]>=16?2:1; + if (model.characterCount==0xffU) return 1; + uint16_t tmp=uint16_t(model.characterCount)*2+2; + if (model.characterCount && tmp>=model.frequencyTotal) + { + value=int32_t(value)*tmp/model.frequencyTotal; + if (model.characterCount+1==model.frequencyTotal) value+=tmp>>2; + } + if (!value) value++; + return value; + }; + + int16_t freq=0,escapeFreq=0; + int16_t currentFrequency=0,frequencyTotal=0; + auto decodeCf=[&](uint16_t shift,bool cmCondition)->uint16_t + { + freq<<=shift; + uint16_t i,value=arithDecoder.decode(freq+escapeFreq)>>shift; + uint32_t lb=0; + for (i=index;i!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)) + { + auto &frequency=frequencies[i]; + if (cmCondition||!characterMask[frequency.character]) + { + if (frequencyTotal+frequency.frequency<=value) + { + frequencyTotal+=frequency.frequency; + } else { + currentFrequency=frequency.frequency<<shift; + break; + } + } + } + frequencyTotal<<=shift; + return i; + }; + + auto decodeCh=[&](uint16_t chPos,uint16_t insertPos,bool cmCondition)->bool + { + if (chPos==0xffffU) + { + arithDecoder.scale(freq,freq+escapeFreq,freq+escapeFreq); + if (models[index].frequencyTotal==1 && initialEscapeChar[models[index].contextLength]<32) + initialEscapeChar[models[index].contextLength]++; + uint16_t prevI=0; + for (uint16_t lb=0,i=index;i!=0xffffU;prevI=i,i=frequencies[i].next,loopBreaker(lb++)) + { + auto &frequency=frequencies[i]; + if (cmCondition||!characterMask[frequency.character]) + { + if (characterMaskStackPointer==256) throw Decompressor::DecompressionError(); + characterMaskStack[characterMaskStackPointer++]=frequency.character; + characterMask[frequency.character]=true; + } + } + contextPointer[insertPos]=index|0x8000U; + frequencyArrayIndex[insertPos]=prevI; + ch=256; + return true; + } else { + arithDecoder.scale(frequencyTotal,frequencyTotal+currentFrequency,freq+escapeFreq); + if (models[index].frequencyTotal==1 && initialEscapeChar[models[index].contextLength]) + initialEscapeChar[models[index].contextLength]--; + contextPointer[insertPos]=index; + frequencyArrayIndex[insertPos]=chPos; + ch=frequencies[chPos].character; + return false; + } + }; + + if (characterMaskStackPointer) + { + for (uint16_t lb=0,i=index;i!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)) + { + auto &frequency=frequencies[i]; + if (!characterMask[frequency.character]) + { + freq+=frequency.frequency; + if (frequency.frequency<3) escapeFreq++; + } + } + escapeFreq=getEscFrequency(escapeFreq,index); + + uint16_t chPos=decodeCf(0,false); + if (stackPointer==5) throw Decompressor::DecompressionError(); + if (!decodeCh(chPos,stackPointer,false)) + { + if (escapeCharacterCounter==10) throw Decompressor::DecompressionError(); + escapeCharacterCounter++; + } + stackPointer++; + + } else { + freq=models[index].frequencyTotal; + escapeFreq=getEscFrequency(models[index].escapeFrequency,index); + + uint16_t chPos=decodeCf((escapeCharacterCounter>=5)?(freq<5 && escapeCharacterCounter==10)?2:1:0,true); + + stackPointer=1; + if (decodeCh(chPos,0,true)) + { + escapeCharacterCounter=0; + } else { + if (escapeCharacterCounter<10) escapeCharacterCounter++; + } + } + + if (ch!=256) + { + if (index!=firstExpiry) + { + auto &model=models[index]; + if (index==lastExpiry) + { + lastExpiry=model.expiryPrevious; + } else { + models[model.expiryNext].expiryPrevious=model.expiryPrevious; + models[model.expiryPrevious].expiryNext=model.expiryNext; + } + models[firstExpiry].expiryPrevious=index; + model.expiryNext=firstExpiry; + firstExpiry=index; + } + break; + } + } + + if (ch==256) break; + + while (stackPointer) + { + uint16_t freqIndex=frequencyArrayIndex[--stackPointer]; + uint16_t pointer=contextPointer[stackPointer]; + auto &model=models[pointer&0x7fffU]; + if (pointer&0x8000U) + { + if (freeBlockPointer==0xffffU) + { + for (uint16_t i=0;i<=stackPointer;) + { + // yuck + uint32_t lb=0; + do + { + releaseBlock=(releaseBlock!=9999U)?releaseBlock+1:0; + loopBreaker(lb++); + } while (frequencies[releaseBlock].next==0xffffU); + for (i=0;i<=stackPointer;i++) + if ((contextPointer[i]&0x7fffU)==releaseBlock) break; + } + auto &frequencyRB=frequencies[releaseBlock]; + auto &modelRB=models[releaseBlock]; + uint16_t f=frequencyRB.frequency; + for (uint16_t lb=0,i=frequencyRB.next;i!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)) + if (frequencies[i].frequency<f) f=frequencies[i].frequency; + bool stopProcess=false; + if (frequencyRB.frequency<++f) + { + uint16_t i,lb=0; + for (lb=0,i=frequencyRB.next;frequencies[i].frequency<f&&frequencies[i].next!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)); + auto &frequencyI=frequencies[i]; + frequencyRB.frequency=frequencyI.frequency; + frequencyRB.character=frequencyI.character; + uint16_t j=frequencyI.next; + frequencyI.next=freeBlockPointer; + freeBlockPointer=frequencyRB.next; + frequencyRB.next=j; + if (j==0xffffU) + { + modelRB.characterCount=0; + uint16_t tmp=modelRB.frequencyTotal=frequencyRB.frequency; + modelRB.escapeFrequency=tmp<3?1:0; + stopProcess=true; + } + } + if (!stopProcess) + { + { + modelRB.characterCount=0; + uint16_t tmp=frequencyRB.frequency/=f; + modelRB.frequencyTotal=tmp; + modelRB.escapeFrequency=tmp<3?1:0; + } + for (uint16_t lb=0,i=frequencyRB.next,j=releaseBlock;i!=0xffffU;i=frequencies[j].next,loopBreaker(lb++)) + { + if (frequencies[i].frequency<f) + { + frequencies[j].next=frequencies[i].next; + frequencies[i].next=freeBlockPointer; + freeBlockPointer=i; + } else { + modelRB.characterCount++; + uint16_t tmp=frequencies[i].frequency/=f; + if (tmp<3) modelRB.escapeFrequency++; + modelRB.frequencyTotal+=tmp; + j=i; + } + } + } + } + freqIndex=frequencies[freqIndex].next=freeBlockPointer; + freeBlockPointer=frequencies[freeBlockPointer].next; + auto &frequencyFI=frequencies[freqIndex]; + frequencyFI.frequency=1U; + frequencyFI.next=0xffffU; + frequencyFI.character=uint8_t(ch); + model.escapeFrequency++; + model.characterCount++; + } else if (++frequencies[freqIndex].frequency==3) { + model.escapeFrequency--; + } + if ((++model.frequencyTotal)/(model.characterCount+1)>frequencies[freqIndex].frequency*2) + { + model.refreshCounter--; + } else if (model.refreshCounter<4) { + model.refreshCounter++; + } + // one ugly scaler + if (!model.refreshCounter||model.frequencyTotal>=8000) + { + model.refreshCounter++; + model.escapeFrequency=0; + model.frequencyTotal=0; + for (uint16_t lb=0,i=pointer&0x7fffU;i!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)) + { + if (frequencies[i].frequency>1) + { + uint16_t tmp=frequencies[i].frequency>>=1; + model.frequencyTotal+=tmp; + if (tmp<3) model.escapeFrequency++; + } else { + model.escapeFrequency++; + model.frequencyTotal++; + } + } + } + } + + uint8_t maxLength=maxContextLength+1; + while (maxLength-->minLength) + { + auto hash=[&](const uint8_t *block,uint32_t length)->uint16_t + { + uint16_t ret=0; + for (uint32_t i=0;i<length;i++) + ret=hashes[(block[i]+ret)&0x3fffU].random; + return ret; + }; + uint16_t le=lastExpiry; + auto &model=models[le]; + auto &frequency=frequencies[le]; + lastExpiry=model.expiryPrevious; + + models[firstExpiry].expiryPrevious=le; + model.expiryNext=firstExpiry; + firstExpiry=le; + if (model.contextLength!=0xffU) + { + if (model.contextLength==4 && !--dropCount) + maxContextLength=3; + uint16_t h=hash(model.context,model.contextLength); + if (hashes[h].data==le) + { + hashes[h].data=model.hashPointer; + } else { + uint16_t i=hashes[h].data; + uint32_t lb=0; + while (le!=models[i].hashPointer) + { + i=models[i].hashPointer; + loopBreaker(lb++); + } + models[i].hashPointer=model.hashPointer; + } + if (frequency.next!=0xffffU) + { + uint16_t i=frequency.next; + uint32_t lb=0; + while (frequencies[i].next!=0xffffU) + { + i=frequencies[i].next; + loopBreaker(lb++); + } + frequencies[i].next=freeBlockPointer; + freeBlockPointer=frequency.next; + } + } + frequency.next=0xffffU; + frequency.frequency=1; + frequency.character=uint8_t(ch); + model.escapeFrequency=1; + model.frequencyTotal=1; + model.contextLength=maxLength; + model.characterCount=0; + model.refreshCounter=4; + for (uint32_t i=0;i<4;i++) model.context[i]=currentContext[i]; + uint16_t hashValue=hash(currentContext,maxLength); + model.hashPointer=hashes[hashValue].data; + hashes[hashValue].data=le; + } + + outputStream.writeByte(uint8_t(ch)); + for (uint16_t i=3;i!=0;i--) + { + currentContext[i]=currentContext[i-1]; + } + currentContext[0]=uint8_t(ch); + } + if (!outputStream.eof()) throw Decompressor::DecompressionError(); +} + +void SXSCDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size(),true); + + uint8_t mode=inputStream.readByte(); + + bool needsTmpBuffer=(mode>=2); + std::unique_ptr<MemoryBuffer> tmpBuffer; + if (needsTmpBuffer) tmpBuffer=std::make_unique<MemoryBuffer>(rawData.size()); + if (_isHSC) decompressHSC(needsTmpBuffer?*tmpBuffer:rawData,inputStream); + else decompressASC(needsTmpBuffer?*tmpBuffer:rawData,inputStream); + + // Mono, High byte only + // also includes de-interleaving + auto deltaDecode16BE=[&]() + { + size_t length=rawData.size(); + const uint8_t *src=tmpBuffer->data(); + const uint8_t *midSrc=&src[length>>1]; + uint8_t *dest=rawData.data(); + + uint8_t ctr=0; + for (size_t i=0,j=0;j<length;i++,j+=2) + { + ctr+=src[i]; + dest[j]=ctr; + dest[j+1]=midSrc[i]; + } + if (length&1) dest[length-1]=src[length-1]; + }; + + // Stereo, High byte only + // also includes de-interleaving + auto deltaDecode16LE=[&]() + { + size_t length=rawData.size(); + const uint8_t *src=tmpBuffer->data(); + const uint8_t *midSrc=&src[length>>1]; + uint8_t *dest=rawData.data(); + + uint8_t ctr=0; + for (size_t i=0,j=0;j<length;i++,j+=2) + { + dest[j]=midSrc[i]; + ctr+=src[i]; + dest[j+1]=ctr; + } + if (length&1) dest[length-1]=src[length-1]; + }; + + switch (mode) + { + case 0: + // no delta + break; + + case 1: + DLTADecode::decode(rawData,rawData,0,rawData.size()); + break; + + case 2: + deltaDecode16BE(); + break; + + case 3: + deltaDecode16LE(); + break; + + default: + throw Decompressor::DecompressionError(); + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/SXSCDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SXSCDecompressor.hpp new file mode 100644 index 00000000..9e3b3a3d --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/SXSCDecompressor.hpp @@ -0,0 +1,51 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef SXSCDECOMPRESSOR_HPP +#define SXSCDECOMPRESSOR_HPP + +#include <cstdint> + +#include "XPKDecompressor.hpp" +#include "InputStream.hpp" +#include "RangeDecoder.hpp" + +namespace ancient::internal +{ + +class SXSCDecompressor : public XPKDecompressor +{ +public: + SXSCDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~SXSCDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + class SXSCReader : public RangeDecoder::BitReader + { + public: + SXSCReader(ForwardInputStream &stream); + virtual ~SXSCReader(); + + virtual uint32_t readBit() override final; + + private: + MSBBitReader<ForwardInputStream> _reader; + }; + + void decompressASC(Buffer &rawData,ForwardInputStream &inputStream); + void decompressHSC(Buffer &rawData,ForwardInputStream &inputStream); + + const Buffer &_packedData; + bool _isHSC; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/StoneCrackerDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/StoneCrackerDecompressor.cpp new file mode 100644 index 00000000..2410f105 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/StoneCrackerDecompressor.cpp @@ -0,0 +1,695 @@ +/* Copyright (C) Teemu Suutari */ + +#include "StoneCrackerDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "HuffmanDecoder.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool StoneCrackerDecompressor::detectHeaderAndGeneration(uint32_t hdr,uint32_t &generation) noexcept +{ + // Stonecracker 2.71 (and others from 2.69 - 2.8 series) do not have any sensible identification value + // first 3 bytes are constants for RLE-encoder (byte values for RLE-encoder, at least they are unique) + // last byte is bit length. + // + // 2.92 and 2.99 do not have either any proper identification word either, however its + // bit lengts for decompressor are stored in the first 4 bytes, which forms identifiable + // value. + // + // Thus for detecting 2.71 and friends, we are creating lots of false positives here. + // At least we can rule those out later when detecting the actual header content + // Final complication is that values for 2.71/2.9X overlap, this we need to handle + // later as well + if (hdr>=0x08090a08U&&hdr<=0x08090a0eU&&hdr!=0x08090a09U) + { + // can be generation 1 as well. needs to be determined later + generation=2; + return true; + } + if ((hdr&0xffU)>=0x08U&&(hdr&0xffU)<=0x0eU) + { + uint8_t byte0=hdr>>24U; + uint8_t byte1=hdr>>16U; + uint8_t byte2=hdr>>8U; + // only limiter I can think of apart from the last byte is that the rle-bytes are unique + if (byte0!=byte1 && byte0!=byte2 && byte1!=byte2) + { + generation=1; + return true; + } + } + // From 3.00 and onwards we can be certain of the format + switch (hdr) + { + case FourCC("S300"): + generation=3; + return true; + + case FourCC("S310"): + generation=4; + return true; + + case FourCC("S400"): + generation=5; + return true; + + case FourCC("S401"): + generation=6; + return true; + + case FourCC("S403"): + generation=7; + return true; + + case FourCC("S404"): + generation=8; + return true; + + default: + return false; + } +} + +bool StoneCrackerDecompressor::detectHeader(uint32_t hdr) noexcept +{ + uint32_t dummy; + return detectHeaderAndGeneration(hdr,dummy); +} + +std::shared_ptr<Decompressor> StoneCrackerDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) +{ + return std::make_shared<StoneCrackerDecompressor>(packedData,exactSizeKnown,verify); +} + +void StoneCrackerDecompressor::initialize(const Buffer &packedData,uint32_t hdr) +{ + auto readModes=[&](uint32_t value) + { + for (uint32_t i=0;i<4U;i++) + { + _modes[i]=value>>24U; + if (_modes[i]<8U || _modes[i]>14U) + throw InvalidFormatError(); + value<<=8U; + } + }; + + switch (_generation) + { + case 1: + _dataOffset=18U; + _rle[0]=hdr>>24U; + _rle[1]=hdr>>16U; + _rle[2]=hdr>>8U; + _modes[0]=hdr; + if (packedData.size()<_dataOffset) throw InvalidFormatError(); + for (uint32_t i=1;i<3U;i++) + { + _modes[i]=packedData.read8(i+15U); + if (_modes[i]<4U || _modes[i]>7U) + throw InvalidFormatError(); + } + _rleSize=packedData.readBE32(4U); + _rawSize=packedData.readBE32(8U); + _packedSize=packedData.readBE32(12U); + break; + + case 2: + readModes(hdr); + case 4: + case 5: + case 6: + _dataOffset=12U; + if (packedData.size()<_dataOffset) throw InvalidFormatError(); + _rawSize=packedData.readBE32(4U); + _packedSize=packedData.readBE32(8U); + break; + + case 3: + _dataOffset=16U; + if (packedData.size()<_dataOffset) throw InvalidFormatError(); + readModes(packedData.readBE32(4)); + _rawSize=packedData.readBE32(8U); + _packedSize=packedData.readBE32(12U); + break; + + case 7: + case 8: + _dataOffset=16U; + if (packedData.size()<_dataOffset+2U) throw InvalidFormatError(); + _rawSize=packedData.readBE32(8U); + _packedSize=packedData.readBE32(12U)+2U; + break; + + default: + throw InvalidFormatError(); + break; + } + + if (_packedSize>getMaxPackedSize() || _rawSize>getMaxRawSize()) + throw InvalidFormatError(); + + // Final sanity checks on old formats, especially on 2.71 which can still be false positive + // For a sanity check we assume the compressor is actually compressing, which is not a exactly wrong thing to do + // (of course there could be expanding files, but it is doubtful someone would actually used them) + // Also, the compressor seem to crash on large files. Lets cap the filesize to 1M + if (_generation==1 && (_rleSize>_rawSize || _packedSize>_rleSize || _rawSize>1048'576U)) + throw InvalidFormatError(); + _packedSize+=_dataOffset; + if (_packedSize>packedData.size()) + throw InvalidFormatError(); +} + +StoneCrackerDecompressor::StoneCrackerDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : + _packedData(packedData) +{ + uint32_t hdr=packedData.readBE32(0); + if (!detectHeaderAndGeneration(hdr,_generation)) throw InvalidFormatError(); + + bool initialized=false; + // this is the fallback if we have accidentally identified the wrong version + if (_generation==2) + { + try + { + initialize(packedData,hdr); + initialized=true; + } catch (const Error &) { + _generation=1; + } + } + if (!initialized) initialize(packedData,hdr); +} + +StoneCrackerDecompressor::~StoneCrackerDecompressor() +{ + // nothing needed +} + +const std::string &StoneCrackerDecompressor::getName() const noexcept +{ + switch (_generation) + { + case 1: + { + static std::string name="SC: StoneCracker v2.69 - v2.81"; + return name; + } + + case 2: + { + static std::string name="SC: StoneCracker v2.92, v2.99"; + return name; + } + + case 3: + { + static std::string name="S300: StoneCracker v3.00"; + return name; + } + + case 4: + { + static std::string name="S310: StoneCracker v3.10, v3.11b"; + return name; + } + + case 5: + { + static std::string name="S400: StoneCracker pre v4.00"; + return name; + } + + case 6: + { + static std::string name="S401: StoneCracker v4.01"; + return name; + } + + case 7: + { + static std::string name="S403: StoneCracker v4.02a"; + return name; + } + + case 8: + { + static std::string name="S404: StoneCracker v4.10"; + return name; + } + } + static std::string dummy=""; + return dummy; +} + +size_t StoneCrackerDecompressor::getPackedSize() const noexcept +{ + return _packedSize; +} + +size_t StoneCrackerDecompressor::getRawSize() const noexcept +{ + return _rawSize; +} + +// v2.71 +void StoneCrackerDecompressor::decompressGen1(Buffer &rawData) +{ + BackwardInputStream inputStream(_packedData,_dataOffset,_packedSize); + MSBBitReader<BackwardInputStream> bitReader(inputStream); + + MemoryBuffer tmpBuffer(_rleSize); + BackwardOutputStream outputStream(tmpBuffer,0,_rleSize); + + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBitsBE32(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBitsBE32(1); + }; + + + // anchor-bit handling + { + uint32_t value=0; + for (uint32_t i=0;i<4U;i++) + { + value>>=8U; + value|=uint32_t(inputStream.readByte())<<24U; + } + uint32_t tmp=value; + uint32_t count=0; + while (tmp) + { + tmp<<=1; + count++; + } + if (count) count--; + if (count) + bitReader.reset(value>>(32U-count),count); + } + + struct SCItem + { + uint32_t distanceBits; + uint32_t countBits; + uint32_t addition; + bool isLiteral; + }; + + // logic is very hard to un-convolute... + HuffmanDecoder<SCItem> scDecoder + { + HuffmanCode<SCItem>{2,0b000,SCItem{0,3U,0,true}}, + HuffmanCode<SCItem>{3,0b010,SCItem{0,_modes[2]+1U,8U,true}}, + HuffmanCode<SCItem>{3,0b011,SCItem{_modes[0]+1U,_modes[1]+1U,5U,false}}, + HuffmanCode<SCItem>{2,0b010,SCItem{8U,0,2U,false}}, + HuffmanCode<SCItem>{3,0b110,SCItem{9U,0,3U,false}}, + HuffmanCode<SCItem>{3,0b111,SCItem{10U,0,4U,false}} + }; + + while (!outputStream.eof()) + { + const auto &item=scDecoder.decode(readBit); + if (item.isLiteral) + { + uint32_t count=readBits(item.countBits); + if (!item.addition) count=!count?8U:count; // why?!? + else count+=item.addition; + for (uint32_t i=0;i<count;i++) + outputStream.writeByte(readBits(8)); + } else { + uint32_t count=readBits(item.countBits)+item.addition; + uint32_t distance=readBits(item.distanceBits); + outputStream.copy(distance,count); + } + } + + ForwardInputStream finalInputStream(tmpBuffer,0,_rleSize); + ForwardOutputStream finalOutputStream(rawData,0,_rawSize); + while (!finalOutputStream.eof()) + { + uint8_t cmd=finalInputStream.readByte(); + if (cmd==_rle[0]) + { + uint32_t count=uint32_t(finalInputStream.readByte())+1; + if (count==1) finalOutputStream.writeByte(cmd); + else { + uint8_t value=finalInputStream.readByte(); + for (uint32_t i=0;i<=count;i++) finalOutputStream.writeByte(value); + } + } else if (cmd==_rle[1]) { + uint32_t count=uint32_t(finalInputStream.readByte())+1; + if (count==1) finalOutputStream.writeByte(cmd); + else { + for (uint32_t i=0;i<=count;i++) finalOutputStream.writeByte(_rle[2]); + } + } else finalOutputStream.writeByte(cmd); + } +} + +// v2.92, v2.99 +// v3.00 +void StoneCrackerDecompressor::decompressGen23(Buffer &rawData) +{ + BackwardInputStream inputStream(_packedData,_dataOffset,_packedSize); + LSBBitReader<BackwardInputStream> bitReader(inputStream); + + BackwardOutputStream outputStream(rawData,0,_rawSize); + + // anchor-bit handling + { + uint32_t value=0; + for (uint32_t i=0;i<4U;i++) + { + value>>=8U; + value|=uint32_t(inputStream.readByte())<<24U; + } + if (value) for (uint32_t i=31U;i>0;i--) + { + if (value&(1U<<i)) + { + bitReader.reset(value&((1U<<i)-1U),i); + break; + } + } + } + + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBitsBE32(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBitsBE32(1); + }; + + auto readCount=[&](uint32_t threshold,uint32_t bits)->uint32_t + { + uint32_t ret=0; + uint32_t tmp; + do + { + tmp=rotateBits(readBits(bits),bits); + ret+=tmp; + } while (tmp==threshold); + return ret; + }; + + bool gen3=_generation>=3; + while (!outputStream.eof()) + { + if (readBit()) + { + uint32_t count=readCount(7U,3U); + if (gen3) count++; + // for 2.92 zero count could meant count of 65536 + // for 2.99 it would be 4G + // nevertheless it is an error + // 3.00 fixes this by +1 + if (!count) + throw DecompressionError(); + for (uint32_t i=0;i<count;i++) + outputStream.writeByte(readBits(8)); + } else { + uint32_t mode=rotateBits(readBits(2U),2U); + uint32_t count; + uint32_t modeBits=_modes[mode]+1U; + if (mode==3) + { + if (readBit()) { + count=readCount(7U,3U)+5U; + if (gen3) modeBits=8U; + } else count=readCount(127U,7U)+(gen3?5U:19U); + } else { + count=mode+2U; + } + uint32_t distance=rotateBits(readBits(modeBits),modeBits)+1U; + outputStream.copy(distance,count); + } + } +} + +// v3.10, v3.11b +// pads output file, and this new size is also defined as rawSize in header! +// +// pre v4.00, v4.01 +void StoneCrackerDecompressor::decompressGen456(Buffer &rawData) +{ + ForwardInputStream inputStream(_packedData,_dataOffset,_packedSize); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + + ForwardOutputStream outputStream(rawData,0,_rawSize); + + auto readBits=[&](uint32_t count)->uint32_t + { + if (_generation==4) return bitReader.readBitsBE32(count); + else return bitReader.readBitsBE16(count); + }; + auto readBit=[&]()->uint32_t + { + return readBits(1); + }; + + auto readCount=[&](uint32_t threshold,uint32_t bits)->uint32_t + { + uint32_t ret=0; + uint32_t tmp; + do + { + tmp=readBits(bits); + ret+=tmp; + } while (tmp==threshold); + return ret; + }; + + static const uint8_t modes[4]={10U,11U,12U,8U}; + + while (!outputStream.eof()) + { + if (readBit()) + { + uint32_t mode=readBits(2U); + uint32_t distance=readBits(modes[mode]); + // will obviously throw if distance is 0 + uint32_t count; + if (mode>=2U) + { + if (_generation==4) count=readCount(15U,4U)+3U; + else count=readCount(7U,3U)+3U; + } else { + count=mode+3U; + } + // yet another bug + if (count>_rawSize-uint32_t(outputStream.getOffset())) + count=_rawSize-uint32_t(outputStream.getOffset()); + outputStream.copy(distance,count); + } else { + uint32_t count=readCount(7U,3U); + // A regression in 3.10 that is not fixed until 4.01 + if (_generation==6) count++; + if (!count) + throw DecompressionError(); + for (uint32_t i=0;i<count;i++) + outputStream.writeByte(readBits(8)); + } + } +} + +// v4.02a +void StoneCrackerDecompressor::decompressGen7(Buffer &rawData) +{ + BackwardInputStream inputStream(_packedData,_dataOffset,_packedSize-2U); + LSBBitReader<BackwardInputStream> bitReader(inputStream); + + BackwardOutputStream outputStream(rawData,0,_rawSize); + + // incomplete value at start + { + uint16_t bitCount=_packedData.readBE16(_packedSize-2U); + if (bitCount>16U) + throw DecompressionError(); + uint16_t value=inputStream.readByte(); + value|=uint16_t(inputStream.readByte())<<8U; + bitReader.reset(value,bitCount&0xff); + } + + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBitsBE16(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBitsBE16(1); + }; + + while (!outputStream.eof()) + { + if (readBit()) + { + static const uint8_t distanceBits[4]={5U,8U,10U,12U}; + static const uint32_t distanceAdditions[4]={1U,0x21,0x121U,0x521U}; + uint32_t mode=readBits(2U); + uint32_t distance=readBits(distanceBits[mode])+distanceAdditions[mode]; + // christmas tree! + uint32_t count=2U; + if (!readBit()) + { + count++; + if (!readBit()) + { + count++; + if (!readBit()) + { + count++; + uint32_t tmp; + do + { + tmp=readBits(3U); + count+=tmp; + } while (tmp==7U); + } + } + } + outputStream.copy(distance,count); + } else { + outputStream.writeByte(readBits(8)); + } + } +} + +// v4.10 +void StoneCrackerDecompressor::decompressGen8(Buffer &rawData) +{ + BackwardInputStream inputStream(_packedData,_dataOffset,_packedSize-2U); + MSBBitReader<BackwardInputStream> bitReader(inputStream); + + BackwardOutputStream outputStream(rawData,0,_rawSize); + + // incomplete value at start + uint16_t modeBits=0; + { + uint16_t bitCount=_packedData.readBE16(_packedSize-2U)&15U; + uint16_t value=inputStream.readByte(); + value|=uint16_t(inputStream.readByte())<<8U; + bitReader.reset(value>>(16U-bitCount),bitCount&0xff); + + modeBits=inputStream.readByte(); + modeBits|=uint16_t(inputStream.readByte())<<8U; + if (modeBits<10U || modeBits>14U) + throw DecompressionError(); + } + + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBitsBE16(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBitsBE16(1); + }; + + struct CountItem + { + uint32_t bits; + uint32_t addition; + bool isLiteral; + }; + + struct DistanceItem + { + uint32_t bits; + uint32_t addition; + }; + + HuffmanDecoder<CountItem> countDecoder + { + HuffmanCode<CountItem>{1,0b00000000,CountItem{0,1U,true}}, + HuffmanCode<CountItem>{2,0b00000011,CountItem{1U,2U,false}}, + HuffmanCode<CountItem>{3,0b00000101,CountItem{2U,4U,false}}, + HuffmanCode<CountItem>{4,0b00001000,CountItem{8U,23U,false}}, + HuffmanCode<CountItem>{5,0b00010010,CountItem{3U,8U,false}}, + HuffmanCode<CountItem>{6,0b00100110,CountItem{2U,16U,false}}, + HuffmanCode<CountItem>{7,0b01001110,CountItem{1U,20U,false}}, + HuffmanCode<CountItem>{8,0b10011110,CountItem{0,22U,false}}, + HuffmanCode<CountItem>{8,0b10011111,CountItem{5U,14U,true}} + }; + + HuffmanDecoder<DistanceItem> distanceDecoder + { + HuffmanCode<DistanceItem>{1,0b01,DistanceItem{modeBits,0x221U}}, + HuffmanCode<DistanceItem>{2,0b00,DistanceItem{9U,0x21U}}, + HuffmanCode<DistanceItem>{2,0b01,DistanceItem{5U,0x1U}} + }; + + while (!outputStream.eof()) + { + const auto &countItem=countDecoder.decode(readBit); + uint32_t count=countItem.addition; + uint32_t tmp; + do + { + tmp=readBits(countItem.bits); + count+=tmp; + } while (tmp==0xffU); + if (countItem.isLiteral) + { + for (uint32_t i=0;i<count;i++) + outputStream.writeByte(readBits(8)); + } else { + const auto &distanceItem=distanceDecoder.decode(readBit); + uint32_t distance=readBits(distanceItem.bits)+distanceItem.addition; + outputStream.copy(distance,count); + } + } +} + +void StoneCrackerDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + if (rawData.size()<_rawSize) throw DecompressionError(); + if (!_rawSize) return; + + // The algorithms are all simple LZ compressors. However they will differ from version to version + // on which order they read data, how much and on some other details + // some of them can be nicely combined, others not so. + // + // To the creators credit, you can see generally improving performance as the versions increase + // Although it is still limited by the simple nature of the implementation and does not really + // compare to the more generic data compression algorithms + switch (_generation) + { + case 1: + decompressGen1(rawData); + break; + + case 2: + case 3: + decompressGen23(rawData); + break; + + case 4: + case 5: + case 6: + decompressGen456(rawData); + break; + + case 7: + decompressGen7(rawData); + break; + + case 8: + decompressGen8(rawData); + break; + + default: + throw DecompressionError(); + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/StoneCrackerDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/StoneCrackerDecompressor.hpp new file mode 100644 index 00000000..74045db9 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/StoneCrackerDecompressor.hpp @@ -0,0 +1,52 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef STONECRACKERDECOMPRESSOR_HPP +#define STONECRACKERDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class StoneCrackerDecompressor : public Decompressor +{ +public: + StoneCrackerDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); + + virtual ~StoneCrackerDecompressor(); + + virtual const std::string &getName() const noexcept override final; + + virtual size_t getPackedSize() const noexcept override final; + virtual size_t getRawSize() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + + static bool detectHeader(uint32_t hdr) noexcept; + + static std::shared_ptr<Decompressor> create(const Buffer &packedData,bool exactSizeKnown,bool verify); + +private: + static bool detectHeaderAndGeneration(uint32_t hdr,uint32_t &generation) noexcept; + + void initialize(const Buffer &packedData,uint32_t hdr); + void decompressGen1(Buffer &rawData); + void decompressGen23(Buffer &rawData); + void decompressGen456(Buffer &rawData); + void decompressGen7(Buffer &rawData); + void decompressGen8(Buffer &rawData); + + const Buffer &_packedData; + + uint32_t _rawSize=0; + uint32_t _packedSize=0; + uint32_t _rleSize=0; + uint8_t _modes[4]; + uint8_t _rle[3]; + uint32_t _generation; + uint32_t _dataOffset; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/TDCSDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/TDCSDecompressor.cpp new file mode 100644 index 00000000..cae05c0e --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/TDCSDecompressor.cpp @@ -0,0 +1,96 @@ +/* Copyright (C) Teemu Suutari */ + +#include "TDCSDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool TDCSDecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("TDCS"); +} + +std::shared_ptr<XPKDecompressor> TDCSDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<TDCSDecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +TDCSDecompressor::TDCSDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); +} + +TDCSDecompressor::~TDCSDecompressor() +{ + // nothing needed +} + +const std::string &TDCSDecompressor::getSubName() const noexcept +{ + static std::string name="XPK-TDCS: LZ77-compressor"; + return name; +} + +void TDCSDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto read2Bits=[&]()->uint32_t + { + return bitReader.readBitsBE32(2); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + while (!outputStream.eof()) + { + uint32_t distance=0; + uint32_t count=0; + uint32_t tmp; + switch (read2Bits()) + { + case 0: + outputStream.writeByte(readByte()); + break; + + case 1: + tmp=uint32_t(readByte())<<8; + tmp|=uint32_t(readByte()); + count=(tmp&3)+3; + distance=((tmp>>2)^0x3fff)+1; + break; + + case 2: + tmp=uint32_t(readByte())<<8; + tmp|=uint32_t(readByte()); + count=(tmp&0xf)+3; + distance=((tmp>>4)^0xfff)+1; + break; + + case 3: + distance=uint32_t(readByte())<<8; + distance|=uint32_t(readByte()); + count=uint32_t(readByte())+3; + if (!distance) throw Decompressor::DecompressionError(); + distance=(distance^0xffff)+1; + break; + + default: + throw Decompressor::DecompressionError(); + } + if (count && distance) + outputStream.copy(distance,count); + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/TDCSDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/TDCSDecompressor.hpp new file mode 100644 index 00000000..edbd22f4 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/TDCSDecompressor.hpp @@ -0,0 +1,31 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef TDCSDECOMPRESSOR_HPP +#define TDCSDECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class TDCSDecompressor : public XPKDecompressor +{ +public: + TDCSDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~TDCSDecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/TPWMDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/TPWMDecompressor.cpp new file mode 100644 index 00000000..c078e465 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/TPWMDecompressor.cpp @@ -0,0 +1,92 @@ +/* Copyright (C) Teemu Suutari */ + +#include "TPWMDecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool TPWMDecompressor::detectHeader(uint32_t hdr) noexcept +{ + return hdr==FourCC("TPWM"); +} + +std::shared_ptr<Decompressor> TPWMDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) +{ + return std::make_shared<TPWMDecompressor>(packedData,verify); +} + +TPWMDecompressor::TPWMDecompressor(const Buffer &packedData,bool verify) : + _packedData(packedData) +{ + uint32_t hdr=packedData.readBE32(0); + if (!detectHeader(hdr) || packedData.size()<12) throw InvalidFormatError(); + + _rawSize=packedData.readBE32(4); + if (!_rawSize || _rawSize>getMaxRawSize()) + throw InvalidFormatError(); +} + +TPWMDecompressor::~TPWMDecompressor() +{ + // nothing needed +} + +const std::string &TPWMDecompressor::getName() const noexcept +{ + static std::string name="TPWM: Turbo Packer"; + return name; +} + +size_t TPWMDecompressor::getPackedSize() const noexcept +{ + // No packed size in the stream :( + // After decompression, we can tell how many bytes were actually used + return _decompressedPackedSize; +} + +size_t TPWMDecompressor::getRawSize() const noexcept +{ + return _rawSize; +} + +void TPWMDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + if (rawData.size()<_rawSize) throw DecompressionError(); + + ForwardInputStream inputStream(_packedData,8,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + auto readByte=[&]()->uint8_t + { + return inputStream.readByte(); + }; + + ForwardOutputStream outputStream(rawData,0,_rawSize); + + while (!outputStream.eof()) + { + if (readBit()) + { + uint8_t byte1=readByte(); + uint8_t byte2=readByte(); + uint32_t distance=(uint32_t(byte1&0xf0)<<4)|byte2; + uint32_t count=uint32_t(byte1&0xf)+3; + + count=std::min(count,uint32_t(_rawSize-outputStream.getOffset())); + outputStream.copy(distance,count); + } else { + outputStream.writeByte(readByte()); + } + } + + _decompressedPackedSize=inputStream.getOffset(); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/TPWMDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/TPWMDecompressor.hpp new file mode 100644 index 00000000..5fd74a95 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/TPWMDecompressor.hpp @@ -0,0 +1,36 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef TPWMDECOMPRESSOR_HPP +#define TPWMDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class TPWMDecompressor : public Decompressor +{ +public: + TPWMDecompressor(const Buffer &packedData,bool verify); + + virtual ~TPWMDecompressor(); + + virtual const std::string &getName() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + virtual size_t getRawSize() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + + static bool detectHeader(uint32_t hdr) noexcept; + static std::shared_ptr<Decompressor> create(const Buffer &packedData,bool exactSizeKnown,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _rawSize=0; + size_t _decompressedPackedSize=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/XPKDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/XPKDecompressor.cpp new file mode 100644 index 00000000..be40d5f9 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/XPKDecompressor.cpp @@ -0,0 +1,25 @@ +/* Copyright (C) Teemu Suutari */ + +#include "XPKDecompressor.hpp" +#include "XPKMain.hpp" + +namespace ancient::internal +{ + +XPKDecompressor::State::~State() +{ + // nothing needed +} + +XPKDecompressor::XPKDecompressor(uint32_t recursionLevel) : + _recursionLevel(recursionLevel) +{ + // nothing needed +} + +XPKDecompressor::~XPKDecompressor() +{ + // nothing needed +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/XPKDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/XPKDecompressor.hpp new file mode 100644 index 00000000..26e1d920 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/XPKDecompressor.hpp @@ -0,0 +1,48 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef XPKDECOMPRESSOR_HPP +#define XPKDECOMPRESSOR_HPP + +#include <cstddef> +#include <cstdint> + +#include <string> + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class XPKDecompressor +{ +public: + class State + { + public: + State(const State&)=delete; + State& operator=(const State&)=delete; + + State()=default; + virtual ~State(); + + uint32_t getRecursionLevel() const; + }; + + XPKDecompressor(const XPKDecompressor&)=delete; + XPKDecompressor& operator=(const XPKDecompressor&)=delete; + + XPKDecompressor(uint32_t recursionLevel=0); + virtual ~XPKDecompressor(); + + virtual const std::string &getSubName() const noexcept=0; + + // Actual decompression + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify)=0; + +protected: + uint32_t _recursionLevel; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/XPKMain.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/XPKMain.cpp new file mode 100644 index 00000000..7b3089f0 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/XPKMain.cpp @@ -0,0 +1,342 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstring> +#include <memory> +#include <algorithm> + +#include "common/SubBuffer.hpp" +#include "common/OverflowCheck.hpp" +#include "common/Common.hpp" +#include "common/Common.hpp" +#include "XPKMain.hpp" +#include "XPKDecompressor.hpp" + +#include "ACCADecompressor.hpp" +#include "ARTMDecompressor.hpp" +#include "BLZWDecompressor.hpp" +#include "BZIP2Decompressor.hpp" +#include "CBR0Decompressor.hpp" +#include "CRMDecompressor.hpp" +#include "CYB2Decoder.hpp" +#include "DEFLATEDecompressor.hpp" +#include "DLTADecode.hpp" +#include "FASTDecompressor.hpp" +#include "FBR2Decompressor.hpp" +#include "FRLEDecompressor.hpp" +#include "HFMNDecompressor.hpp" +#include "HUFFDecompressor.hpp" +#include "ILZRDecompressor.hpp" +#include "IMPDecompressor.hpp" +#include "LHLBDecompressor.hpp" +#include "LIN1Decompressor.hpp" +#include "LIN2Decompressor.hpp" +#include "LZBSDecompressor.hpp" +#include "LZCBDecompressor.hpp" +#include "LZW2Decompressor.hpp" +#include "LZW4Decompressor.hpp" +#include "LZW5Decompressor.hpp" +#include "LZXDecompressor.hpp" +#include "MASHDecompressor.hpp" +#include "NONEDecompressor.hpp" +#include "NUKEDecompressor.hpp" +#include "PPDecompressor.hpp" +#include "RAKEDecompressor.hpp" +#include "RDCNDecompressor.hpp" +#include "RLENDecompressor.hpp" +#include "SDHCDecompressor.hpp" +#include "SHR3Decompressor.hpp" +#include "SHRIDecompressor.hpp" +#include "SLZ3Decompressor.hpp" +#include "SMPLDecompressor.hpp" +#include "SQSHDecompressor.hpp" +#include "SXSCDecompressor.hpp" +#include "TDCSDecompressor.hpp" +#include "ZENODecompressor.hpp" + +namespace ancient::internal +{ + +bool XPKMain::detectHeader(uint32_t hdr) noexcept +{ + return hdr==FourCC("XPKF"); +} + +std::shared_ptr<Decompressor> XPKMain::create(const Buffer &packedData,bool verify,bool exactSizeKnown) +{ + return std::make_shared<XPKMain>(packedData,verify,0); +} + +static std::vector<std::pair<bool(*)(uint32_t),std::shared_ptr<XPKDecompressor>(*)(uint32_t,uint32_t,const Buffer&,std::shared_ptr<XPKDecompressor::State>&,bool)>> XPKDecompressors={ + {ACCADecompressor::detectHeaderXPK,ACCADecompressor::create}, + {ARTMDecompressor::detectHeaderXPK,ARTMDecompressor::create}, + {BLZWDecompressor::detectHeaderXPK,BLZWDecompressor::create}, + {BZIP2Decompressor::detectHeaderXPK,BZIP2Decompressor::create}, + {CBR0Decompressor::detectHeaderXPK,CBR0Decompressor::create}, + {CRMDecompressor::detectHeaderXPK,CRMDecompressor::create}, + {CYB2Decoder::detectHeaderXPK,CYB2Decoder::create}, + {DEFLATEDecompressor::detectHeaderXPK,DEFLATEDecompressor::create}, + {DLTADecode::detectHeaderXPK,DLTADecode::create}, + {FASTDecompressor::detectHeaderXPK,FASTDecompressor::create}, + {FBR2Decompressor::detectHeaderXPK,FBR2Decompressor::create}, + {FRLEDecompressor::detectHeaderXPK,FRLEDecompressor::create}, + {HFMNDecompressor::detectHeaderXPK,HFMNDecompressor::create}, + {HUFFDecompressor::detectHeaderXPK,HUFFDecompressor::create}, + {ILZRDecompressor::detectHeaderXPK,ILZRDecompressor::create}, + {IMPDecompressor::detectHeaderXPK,IMPDecompressor::create}, + {LHLBDecompressor::detectHeaderXPK,LHLBDecompressor::create}, + {LIN1Decompressor::detectHeaderXPK,LIN1Decompressor::create}, + {LIN2Decompressor::detectHeaderXPK,LIN2Decompressor::create}, + {LZBSDecompressor::detectHeaderXPK,LZBSDecompressor::create}, + {LZCBDecompressor::detectHeaderXPK,LZCBDecompressor::create}, + {LZW2Decompressor::detectHeaderXPK,LZW2Decompressor::create}, + {LZW4Decompressor::detectHeaderXPK,LZW4Decompressor::create}, + {LZW5Decompressor::detectHeaderXPK,LZW5Decompressor::create}, + {LZXDecompressor::detectHeaderXPK,LZXDecompressor::create}, + {MASHDecompressor::detectHeaderXPK,MASHDecompressor::create}, + {NONEDecompressor::detectHeaderXPK,NONEDecompressor::create}, + {NUKEDecompressor::detectHeaderXPK,NUKEDecompressor::create}, + {PPDecompressor::detectHeaderXPK,PPDecompressor::create}, + {RAKEDecompressor::detectHeaderXPK,RAKEDecompressor::create}, + {RDCNDecompressor::detectHeaderXPK,RDCNDecompressor::create}, + {RLENDecompressor::detectHeaderXPK,RLENDecompressor::create}, + {SDHCDecompressor::detectHeaderXPK,SDHCDecompressor::create}, + {SHR3Decompressor::detectHeaderXPK,SHR3Decompressor::create}, + {SHRIDecompressor::detectHeaderXPK,SHRIDecompressor::create}, + {SLZ3Decompressor::detectHeaderXPK,SLZ3Decompressor::create}, + {SMPLDecompressor::detectHeaderXPK,SMPLDecompressor::create}, + {SQSHDecompressor::detectHeaderXPK,SQSHDecompressor::create}, + {SXSCDecompressor::detectHeaderXPK,SXSCDecompressor::create}, + {TDCSDecompressor::detectHeaderXPK,TDCSDecompressor::create}, + {ZENODecompressor::detectHeaderXPK,ZENODecompressor::create}}; + +XPKMain::XPKMain(const Buffer &packedData,bool verify,uint32_t recursionLevel) : + _packedData(packedData) +{ + if (packedData.size()<44) throw InvalidFormatError(); + uint32_t hdr=packedData.readBE32(0); + if (!detectHeader(hdr)) throw InvalidFormatError(); + + _packedSize=packedData.readBE32(4); + _type=packedData.readBE32(8); + _rawSize=packedData.readBE32(12); + + if (!_rawSize || !_packedSize) throw InvalidFormatError(); + if (_rawSize>getMaxRawSize() || _packedSize>getMaxPackedSize()) throw InvalidFormatError(); + + uint8_t flags=packedData.read8(32); + _longHeaders=(flags&1)?true:false; + if (flags&2) throw InvalidFormatError(); // needs password. we do not support that + if (flags&4) // extra header + { + _headerSize=38+uint32_t(packedData.readBE16(36)); + } else { + _headerSize=36; + } + + if (OverflowCheck::sum(_packedSize,8U)>packedData.size()) throw InvalidFormatError(); + + bool found=false; + for (auto &it : XPKDecompressors) + { + if (it.first(_type)) + { + if (recursionLevel>=getMaxRecursionLevel()) throw InvalidFormatError(); + else { + found=true; + break; + } + } + } + if (!found) throw InvalidFormatError(); + + auto headerChecksum=[](const Buffer &buffer,size_t offset,size_t len)->bool + { + if (!len || OverflowCheck::sum(offset,len)>buffer.size()) return false; + const uint8_t *ptr=buffer.data()+offset; + uint8_t tmp=0; + for (size_t i=0;i<len;i++) + tmp^=ptr[i]; + return !tmp; + }; + + // this implementation assumes align padding is zeros + auto chunkChecksum=[](const Buffer &buffer,size_t offset,size_t len,uint16_t checkValue)->bool + { + if (!len || OverflowCheck::sum(offset,len)>buffer.size()) return false; + const uint8_t *ptr=buffer.data()+offset; + uint8_t tmp[2]={0,0}; + for (size_t i=0;i<len;i++) + tmp[i&1]^=ptr[i]; + return tmp[0]==(checkValue>>8) && tmp[1]==(checkValue&0xff); + }; + + + if (verify) + { + if (!headerChecksum(_packedData,0,36)) throw VerificationError(); + + std::shared_ptr<XPKDecompressor::State> state; + forEachChunk([&](const Buffer &header,const Buffer &chunk,uint32_t rawChunkSize,uint8_t chunkType)->bool + { + if (!headerChecksum(header,0,header.size())) throw VerificationError(); + + uint16_t hdrCheck=header.readBE16(2); + if (chunk.size() && !chunkChecksum(chunk,0,chunk.size(),hdrCheck)) throw VerificationError(); + + if (chunkType==1) + { + auto sub=createDecompressor(_type,_recursionLevel,chunk,state,true); + } else if (chunkType!=0 && chunkType!=15) throw InvalidFormatError(); + return true; + }); + } +} + +XPKMain::~XPKMain() +{ + // nothing needed +} + +const std::string &XPKMain::getName() const noexcept +{ + std::shared_ptr<XPKDecompressor> sub; + std::shared_ptr<XPKDecompressor::State> state; + try + { + forEachChunk([&](const Buffer &header,const Buffer &chunk,uint32_t rawChunkSize,uint8_t chunkType)->bool + { + try + { + sub=createDecompressor(_type,_recursionLevel,chunk,state,false); + } catch (const Error&) { + // should not happen since the code is already tried out, + // however, lets handle the case gracefully + } + return false; + }); + } catch (const Buffer::Error&) { + // ditto + } + static std::string invName="<invalid>"; + return (sub)?sub->getSubName():invName; +} + +size_t XPKMain::getPackedSize() const noexcept +{ + return _packedSize+8; +} + +size_t XPKMain::getRawSize() const noexcept +{ + return _rawSize; +} + +void XPKMain::decompressImpl(Buffer &rawData,bool verify) +{ + if (rawData.size()<_rawSize) throw DecompressionError(); + + uint32_t destOffset=0; + std::shared_ptr<XPKDecompressor::State> state; + forEachChunk([&](const Buffer &header,const Buffer &chunk,uint32_t rawChunkSize,uint8_t chunkType)->bool + { + if (OverflowCheck::sum(destOffset,rawChunkSize)>rawData.size()) throw DecompressionError(); + if (!rawChunkSize) return true; + + ConstSubBuffer previousBuffer(rawData,0,destOffset); + SubBuffer DestBuffer(rawData,destOffset,rawChunkSize); + switch (chunkType) + { + case 0: + if (rawChunkSize!=chunk.size()) throw DecompressionError();; + std::memcpy(DestBuffer.data(),chunk.data(),rawChunkSize); + break; + + case 1: + { + try + { + auto sub=createDecompressor(_type,_recursionLevel,chunk,state,false); + sub->decompressImpl(DestBuffer,previousBuffer,verify); + } catch (const InvalidFormatError&) { + // we should throw a correct error + throw DecompressionError(); + } + } + break; + + case 15: + break; + + default: + return false; + } + + destOffset+=rawChunkSize; + return true; + }); + + if (destOffset!=_rawSize) throw DecompressionError(); + + if (verify) + { + if (std::memcmp(_packedData.data()+16,rawData.data(),std::min(_rawSize,16U))) throw DecompressionError(); + } +} + +std::shared_ptr<XPKDecompressor> XPKMain::createDecompressor(uint32_t type,uint32_t recursionLevel,const Buffer &buffer,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + // since this method is used externally, better check recursion level + if (recursionLevel>=getMaxRecursionLevel()) throw InvalidFormatError(); + for (auto &it : XPKDecompressors) + { + if (it.first(type)) return it.second(type,recursionLevel,buffer,state,verify); + } + throw InvalidFormatError(); +} + +template <typename F> +void XPKMain::forEachChunk(F func) const +{ + uint32_t currentOffset=0,rawSize,packedSize; + bool isLast=false; + + while (currentOffset<_packedSize+8 && !isLast) + { + auto readDualValue=[&](uint32_t offsetShort,uint32_t offsetLong,uint32_t &value) + { + if (_longHeaders) + { + value=_packedData.readBE32(currentOffset+offsetLong); + } else { + value=uint32_t(_packedData.readBE16(currentOffset+offsetShort)); + } + }; + + uint32_t chunkHeaderLen=_longHeaders?12:8; + if (!currentOffset) + { + // return first; + currentOffset=_headerSize; + } else { + uint32_t tmp; + readDualValue(4,4,tmp); + tmp=((tmp+3U)&~3U); + if (OverflowCheck::sum(tmp,currentOffset,chunkHeaderLen)>_packedSize) + throw InvalidFormatError(); + currentOffset+=chunkHeaderLen+tmp; + } + readDualValue(4,4,packedSize); + readDualValue(6,8,rawSize); + + ConstSubBuffer hdr(_packedData,currentOffset,chunkHeaderLen); + ConstSubBuffer chunk(_packedData,currentOffset+chunkHeaderLen,packedSize); + + uint8_t type=_packedData.read8(currentOffset); + if (!func(hdr,chunk,rawSize,type)) return; + + if (type==15) isLast=true; + } + if (!isLast) throw InvalidFormatError(); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/XPKMain.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/XPKMain.hpp new file mode 100644 index 00000000..5ad6c08f --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/XPKMain.hpp @@ -0,0 +1,54 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef XPKMAIN_HPP +#define XPKMAIN_HPP + +#include "Decompressor.hpp" +#include "XPKDecompressor.hpp" + +#include <vector> + +namespace ancient::internal +{ + +class XPKMain : public Decompressor +{ +friend class XPKDecompressor; +public: + XPKMain(const Buffer &packedData,bool verify,uint32_t recursionLevel); + + virtual ~XPKMain(); + + virtual const std::string &getName() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + virtual size_t getRawSize() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + + static bool detectHeader(uint32_t hdr) noexcept; + + static std::shared_ptr<Decompressor> create(const Buffer &packedData,bool exactSizeKnown,bool verify); + + // Can be used to create directly decoder for chunk (needed by CYB2) + static std::shared_ptr<XPKDecompressor> createDecompressor(uint32_t type,uint32_t recursionLevel,const Buffer &buffer,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + static void registerDecompressor(bool(*detect)(uint32_t),std::shared_ptr<XPKDecompressor>(*create)(uint32_t,uint32_t,const Buffer&,std::shared_ptr<XPKDecompressor::State>&,bool)); + static constexpr uint32_t getMaxRecursionLevel() noexcept { return 4; } + + template <typename F> + void forEachChunk(F func) const; + + const Buffer &_packedData; + + uint32_t _packedSize=0; + uint32_t _rawSize=0; + uint32_t _headerSize=0; + uint32_t _type=0; + bool _longHeaders=false; + uint32_t _recursionLevel=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/ZENODecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ZENODecompressor.cpp new file mode 100644 index 00000000..8b2b4e0a --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ZENODecompressor.cpp @@ -0,0 +1,130 @@ +/* Copyright (C) Teemu Suutari */ + +#include "ZENODecompressor.hpp" +#include "InputStream.hpp" +#include "OutputStream.hpp" +#include "common/Common.hpp" + + +namespace ancient::internal +{ + +bool ZENODecompressor::detectHeaderXPK(uint32_t hdr) noexcept +{ + return hdr==FourCC("ZENO"); +} + +std::shared_ptr<XPKDecompressor> ZENODecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) +{ + return std::make_shared<ZENODecompressor>(hdr,recursionLevel,packedData,state,verify); +} + +ZENODecompressor::ZENODecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify) : + XPKDecompressor(recursionLevel), + _packedData(packedData) +{ + if (!detectHeaderXPK(hdr) || _packedData.size()<6) + throw Decompressor::InvalidFormatError(); + // first 4 bytes is checksum for password. It needs to be zero + if (_packedData.readBE32(0)) throw Decompressor::InvalidFormatError(); + _maxBits=_packedData.read8(4); + if (_maxBits<9 || _maxBits>20) throw Decompressor::InvalidFormatError(); + _startOffset=uint32_t(_packedData.read8(5))+6; + if (_startOffset>=_packedData.size()) throw Decompressor::InvalidFormatError(); +} + +ZENODecompressor::~ZENODecompressor() +{ + // nothing needed +} + +const std::string &ZENODecompressor::getSubName() const noexcept +{ + static std::string name="XPK-ZENO: LZW-compressor"; + return name; +} + +void ZENODecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) +{ + ForwardInputStream inputStream(_packedData,_startOffset,_packedData.size()); + MSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + uint32_t maxCode=1<<_maxBits; + uint32_t stackLength=5000; // magic constant + auto prefix=std::make_unique<uint32_t[]>(maxCode-258); + auto suffix=std::make_unique<uint8_t[]>(maxCode-258); + auto stack=std::make_unique<uint8_t[]>(stackLength); + + uint32_t freeIndex,codeBits,prevCode,newCode; + + auto init=[&]() + { + codeBits=9; + freeIndex=258; + }; + + init(); + prevCode=readBits(9); + newCode=prevCode; + suffix[freeIndex-258]=0; + prefix[freeIndex-258]=0; + freeIndex++; + outputStream.writeByte(newCode); + + while (!outputStream.eof()) + { + if (freeIndex+3>=(1U<<codeBits) && codeBits<_maxBits) codeBits++; + uint32_t code=readBits(codeBits); + switch (code) + { + case 256: + throw Decompressor::DecompressionError(); + break; + + case 257: + init(); + break; + + default: + { + uint32_t stackPos=0; + uint32_t tmp=code; + if (tmp==freeIndex) + { + stack[stackPos++]=newCode; + tmp=prevCode; + } + if (tmp>=258) + { + do { + if (stackPos+1>=stackLength || tmp>=freeIndex) throw Decompressor::DecompressionError(); + stack[stackPos++]=suffix[tmp-258]; + tmp=prefix[tmp-258]; + } while (tmp>=258); + stack[stackPos++]=newCode=tmp; + while (stackPos) outputStream.writeByte(stack[--stackPos]); + } else { + newCode=tmp; + outputStream.writeByte(tmp); + if (stackPos) outputStream.writeByte(stack[0]); + } + } + if (freeIndex<maxCode) + { + suffix[freeIndex-258]=newCode; + prefix[freeIndex-258]=prevCode; + freeIndex++; + } + prevCode=code; + break; + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/ZENODecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ZENODecompressor.hpp new file mode 100644 index 00000000..20272e40 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/ZENODecompressor.hpp @@ -0,0 +1,34 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef ZENODECOMPRESSOR_HPP +#define ZENODECOMPRESSOR_HPP + +#include "XPKDecompressor.hpp" + +namespace ancient::internal +{ + +class ZENODecompressor : public XPKDecompressor +{ +public: + ZENODecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + + virtual ~ZENODecompressor(); + + virtual const std::string &getSubName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; + + static bool detectHeaderXPK(uint32_t hdr) noexcept; + static std::shared_ptr<XPKDecompressor> create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr<XPKDecompressor::State> &state,bool verify); + +private: + const Buffer &_packedData; + + uint32_t _maxBits=0; + size_t _startOffset=0; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ImplodeDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ImplodeDecompressor.cpp new file mode 100644 index 00000000..717f01f9 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ImplodeDecompressor.cpp @@ -0,0 +1,130 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstdint> +#include <cstring> + +#include "ImplodeDecompressor.hpp" +#include "../InputStream.hpp" +#include "../OutputStream.hpp" +#include "../HuffmanDecoder.hpp" + + +namespace ancient::internal +{ + +ImplodeDecompressor::ImplodeDecompressor(const Buffer &packedData,uint32_t flags) : + _packedData(packedData), + _flags(flags) +{ + +} + +ImplodeDecompressor::~ImplodeDecompressor() +{ + // nothing needed +} + +size_t ImplodeDecompressor::getRawSize() const noexcept +{ + // N/A + return 0; +} + +size_t ImplodeDecompressor::getPackedSize() const noexcept +{ + // N/A + return 0; +} + +const std::string &ImplodeDecompressor::getName() const noexcept +{ + static std::string name="Zip: Implode"; + return name; +} + +void ImplodeDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + LSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + auto readBit=[&]()->uint32_t + { + return bitReader.readBits8(1); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + // Although Implode uses Fannon-Shano, Huffman decoder can be used + typedef HuffmanDecoder<uint8_t> ImplodeHuffmanDecoder; + + auto createDecoder=[&](ImplodeHuffmanDecoder &dec,uint32_t maxValue) + { + uint32_t length=readBits(8)+1; + uint8_t bitLengths[257]; + uint8_t counts[257]; + uint8_t offsets[257]; + uint8_t minDepth=32,maxDepth=0; + + for (uint32_t i=0;i<length;i++) + { + bitLengths[i]=readBits(4)+1; + counts[i]=readBits(4)+1; + offsets[i]=i?offsets[i-1]+counts[i-1]:0; + if (bitLengths[i]<minDepth) minDepth=bitLengths[i]; + if (bitLengths[i]>maxDepth) maxDepth=bitLengths[i]; + } + if (static_cast<uint32_t>(offsets[length-1]+counts[length-1])>maxValue) throw DecompressionError(); + + uint32_t code=0; + for (uint32_t depth=maxDepth;depth>=minDepth;depth--) + { + for (int32_t i=length-1;i>=0;i--) + { + if (depth!=bitLengths[i]) continue; + for (uint32_t j=counts[i];j>0;j--) + { + dec.insert(HuffmanCode<uint8_t>{depth,code>>(maxDepth-depth),uint8_t(offsets[i]+j-1)}); + code+=1<<(maxDepth-depth); + } + } + } + }; + + ImplodeHuffmanDecoder lengthDecoder,distanceDecoder,litDecoder; + + uint32_t extraDistanceBits=(_flags&2)?7:6; + bool literalDecPresent=_flags&4; + uint32_t lengthAddition=2; + + if (literalDecPresent) + { + createDecoder(litDecoder,256); + lengthAddition++; + } + createDecoder(lengthDecoder,64); + createDecoder(distanceDecoder,64); + + while (!outputStream.eof()) + { + if (readBit()) + { + if (literalDecPresent) outputStream.writeByte(litDecoder.decode(readBit)); + else outputStream.writeByte(readBits(8)); + } else { + uint32_t distance=readBits(extraDistanceBits); + distance|=uint32_t(distanceDecoder.decode(readBit))<<extraDistanceBits; + distance++; + + uint32_t count=lengthDecoder.decode(readBit); + if (count==63) count+=readBits(8); + count+=lengthAddition; + + outputStream.copy(distance,count,0); + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ImplodeDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ImplodeDecompressor.hpp new file mode 100644 index 00000000..8b7cbd4b --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ImplodeDecompressor.hpp @@ -0,0 +1,32 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef IMPLODEDECOMPRESSOR_HPP +#define IMPLODEDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class ImplodeDecompressor : public Decompressor +{ +public: + ImplodeDecompressor(const Buffer &packedData,uint32_t flags); + virtual ~ImplodeDecompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + +private: + const Buffer &_packedData; + + uint32_t _flags; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ReduceDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ReduceDecompressor.cpp new file mode 100644 index 00000000..92459359 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ReduceDecompressor.cpp @@ -0,0 +1,115 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstdint> +#include <cstring> + +#include "ReduceDecompressor.hpp" +#include "../InputStream.hpp" +#include "../OutputStream.hpp" + + +namespace ancient::internal +{ + +ReduceDecompressor::ReduceDecompressor(const Buffer &packedData,uint32_t mode) : + _packedData(packedData), + _mode(mode) +{ + if (mode<2 || mode>5) throw InvalidFormatError(); +} + +ReduceDecompressor::~ReduceDecompressor() +{ + // nothing needed +} + +size_t ReduceDecompressor::getRawSize() const noexcept +{ + // N/A + return 0; +} + +size_t ReduceDecompressor::getPackedSize() const noexcept +{ + // N/A + return 0; +} + +const std::string &ReduceDecompressor::getName() const noexcept +{ + static std::string name="Zip: Reduce"; + return name; +} + +void ReduceDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + LSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + uint8_t SLengths[256]; + auto SBuf=std::make_unique<uint8_t[]>(256*64); + uint8_t (&S)[256][64]=*reinterpret_cast<uint8_t(*)[256][64]>(SBuf.get()); + + static const uint8_t bitLengths[64]={ + 8,1,1,2,2,3,3,3, 3,4,4,4,4,4,4,4, + 4,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5, + 5,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, + 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6}; + + for (int32_t i=255;i>=0;i--) + { + uint32_t length=SLengths[i]=readBits(6); + for (uint32_t j=0;j<length;j++) S[i][j]=readBits(8); + } + + uint8_t lengthBits=9U-_mode; + + uint8_t lastChar=0; + while (!outputStream.eof()) + { + auto readByte=[&]()->uint8_t + { + uint8_t ret; + if (!SLengths[lastChar] || readBits(1)) + { + ret=readBits(8); + } else { + uint8_t tmp=readBits(bitLengths[SLengths[lastChar]]); + if (tmp>=SLengths[lastChar]) throw DecompressionError(); + ret=S[lastChar][tmp]; + } + lastChar=ret; + return ret; + }; + + uint8_t ch=readByte(); + if (ch!=0x90U) + { + outputStream.writeByte(ch); + } else { + ch=readByte(); + if (!ch) + { + outputStream.writeByte(0x90U); + } else { + uint8_t lengthMask=0xffU>>(8U-lengthBits); + uint32_t count=ch&lengthMask; + if (count==lengthMask) + count+=uint32_t(readByte()); + count+=3; + uint32_t distance=(uint32_t(ch>>lengthBits)<<8)|uint32_t(readByte()); + distance++; + + outputStream.copy(distance,count,0); + } + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ReduceDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ReduceDecompressor.hpp new file mode 100644 index 00000000..3612e7eb --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ReduceDecompressor.hpp @@ -0,0 +1,32 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef REDUCEDECOMPRESSOR_HPP +#define REDUCEDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class ReduceDecompressor : public Decompressor +{ +public: + ReduceDecompressor(const Buffer &packedData,uint32_t mode); + virtual ~ReduceDecompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + +private: + const Buffer &_packedData; + + uint32_t _mode; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ShrinkDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ShrinkDecompressor.cpp new file mode 100644 index 00000000..8668e0f6 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ShrinkDecompressor.cpp @@ -0,0 +1,137 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstdint> +#include <cstring> +#include <vector> + +#include "ShrinkDecompressor.hpp" +#include "../InputStream.hpp" +#include "../OutputStream.hpp" + + +namespace ancient::internal +{ + +ShrinkDecompressor::ShrinkDecompressor(const Buffer &packedData) : + _packedData(packedData) +{ + +} + +ShrinkDecompressor::~ShrinkDecompressor() +{ + // nothing needed +} + +size_t ShrinkDecompressor::getRawSize() const noexcept +{ + // N/A + return 0; +} + +size_t ShrinkDecompressor::getPackedSize() const noexcept +{ + // N/A + return 0; +} + +const std::string &ShrinkDecompressor::getName() const noexcept +{ + static std::string name="Zip: Shrink"; + return name; +} + +void ShrinkDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + ForwardInputStream inputStream(_packedData,0,_packedData.size()); + LSBBitReader<ForwardInputStream> bitReader(inputStream); + auto readBits=[&](uint32_t count)->uint32_t + { + return bitReader.readBits8(count); + }; + + ForwardOutputStream outputStream(rawData,0,rawData.size()); + + const uint32_t maxCode=0x2000U; + auto prefix=std::make_unique<uint32_t[]>(maxCode-257); + auto suffix=std::make_unique<uint8_t[]>(maxCode-257); + auto stack=std::make_unique<uint8_t[]>(maxCode-257); + + uint32_t freeIndex,codeBits,prevCode,newCode; + + auto suffixLookup=[&](uint32_t code)->uint32_t + { + // main protection against negative values + if (code>=maxCode) throw DecompressionError(); + return (code<257)?code:suffix[code-257]; + }; + + auto insert=[&](uint32_t code) + { + uint32_t stackPos=0; + newCode=suffixLookup(code); + while (code>=257) + { + if (stackPos+1>=maxCode-257) throw DecompressionError(); + stack[stackPos++]=newCode; + code=prefix[code-257]; + newCode=suffixLookup(code); + } + stack[stackPos++]=newCode; + while (stackPos) outputStream.writeByte(stack[--stackPos]); + }; + + for (uint32_t i=0;i<maxCode-257;i++) prefix[i]=~0U; + codeBits=9; + freeIndex=257; + prevCode=readBits(codeBits); + insert(prevCode); + + while (!outputStream.eof()) + { + uint32_t code=readBits(codeBits); + if (code==256) + { + uint32_t tmp=readBits(codeBits); + if (tmp==1) + { + if (codeBits!=13) codeBits++; + } else if (tmp==2) { + std::vector<bool> usageMap(freeIndex-257,false); + for (uint32_t i=257;i<freeIndex;i++) { + uint32_t tmp=prefix[i-257]; + if (!(tmp&0x8000'0000U) && tmp>=257) usageMap[tmp-257]=true; + } + uint32_t firstEmpty=freeIndex; + for (uint32_t i=257;i<freeIndex;i++) + { + if (!usageMap[i-257]) + { + prefix[i-257]=~0U; + suffix[i-257]=0; + if (i<firstEmpty) firstEmpty=i; + } + } + freeIndex=firstEmpty; + } else throw DecompressionError(); + } else { + if (code>=257 && (prefix[code-257]&0x8000'0000U)) + { + uint32_t tmp=newCode; + insert(prevCode); + outputStream.writeByte(tmp); + } else insert(code); + while (!(prefix[freeIndex-257]&0x8000'0000U) && freeIndex<maxCode) + freeIndex++; + if (freeIndex<maxCode) + { + suffix[freeIndex-257]=newCode; + prefix[freeIndex-257]=prevCode; + freeIndex++; + } + prevCode=code; + } + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ShrinkDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ShrinkDecompressor.hpp new file mode 100644 index 00000000..9c2c7292 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ShrinkDecompressor.hpp @@ -0,0 +1,30 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef SHRINKDECOMPRESSOR_HPP +#define SHRINKDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class ShrinkDecompressor : public Decompressor +{ +public: + ShrinkDecompressor(const Buffer &packedData); + virtual ~ShrinkDecompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + +private: + const Buffer &_packedData; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ZIPDecompressor.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ZIPDecompressor.cpp new file mode 100644 index 00000000..7f0aaaf8 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ZIPDecompressor.cpp @@ -0,0 +1,117 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstdint> +#include <cstring> + +#include "ZIPDecompressor.hpp" + +#include "../BZIP2Decompressor.hpp" +#include "../DEFLATEDecompressor.hpp" +#include "ImplodeDecompressor.hpp" +#include "ReduceDecompressor.hpp" +#include "ShrinkDecompressor.hpp" + + +namespace ancient::internal +{ + +ZIPDecompressor::ZIPDecompressor(const Buffer &packedData,uint32_t method,uint32_t flags) : + _packedData(packedData), + _method(method), + _flags(flags) +{ + +} + +ZIPDecompressor::~ZIPDecompressor() +{ + // nothing needed +} + +size_t ZIPDecompressor::getRawSize() const noexcept +{ + // N/A + return 0; +} + +size_t ZIPDecompressor::getPackedSize() const noexcept +{ + // N/A + return 0; +} + +const std::string &ZIPDecompressor::getName() const noexcept +{ + static std::string name="Zip"; + return name; +} + +void ZIPDecompressor::decompressImpl(Buffer &rawData,bool verify) +{ + switch (_method) + { + case 0: + if (rawData.size()!=_packedData.size()) throw DecompressionError(); + std::memcpy(rawData.data(),_packedData.data(),rawData.size()); + break; + + case 1: + { + ShrinkDecompressor dec(_packedData); + dec.decompress(rawData,verify); + } + break; + + case 2: + case 3: + case 4: + case 5: + { + ReduceDecompressor dec(_packedData,_method); + dec.decompress(rawData,verify); + } + break; + + case 6: + { + ImplodeDecompressor dec(_packedData,_flags); + dec.decompress(rawData,verify); + } + break; + + case 8: + case 9: + { + DEFLATEDecompressor dec(_packedData,_packedData.size(),rawData.size(),false,verify,_method==9); + dec.decompress(rawData,verify); + } + break; + + case 12: + { + BZIP2Decompressor dec(_packedData,true,verify); + dec.decompress(rawData,verify); + } + break; + + case 14: + // LZMA + throw DecompressionError(); + break; + + case 97: + // WavPack + throw DecompressionError(); + break; + + case 98: + // PPMd + throw DecompressionError(); + break; + + default: + throw DecompressionError(); + } +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ZIPDecompressor.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ZIPDecompressor.hpp new file mode 100644 index 00000000..77ecd6fe --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/Zip/ZIPDecompressor.hpp @@ -0,0 +1,33 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef ZIPDECOMPRESSOR_HPP +#define ZIPDECOMPRESSOR_HPP + +#include "Decompressor.hpp" + +namespace ancient::internal +{ + +class ZIPDecompressor : public Decompressor +{ +public: + ZIPDecompressor(const Buffer &packedData,uint32_t method,uint32_t flags); + virtual ~ZIPDecompressor(); + + virtual size_t getRawSize() const noexcept override final; + virtual size_t getPackedSize() const noexcept override final; + + virtual const std::string &getName() const noexcept override final; + + virtual void decompressImpl(Buffer &rawData,bool verify) override final; + +private: + const Buffer &_packedData; + + uint32_t _method; + uint32_t _flags; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/Buffer.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/Buffer.cpp new file mode 100644 index 00000000..0d534a24 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/Buffer.cpp @@ -0,0 +1,112 @@ +/* Copyright (C) Teemu Suutari */ + +#include "Buffer.hpp" +#include "OverflowCheck.hpp" + + +namespace ancient::internal +{ + +Buffer::Error::Error() noexcept +{ + // nothing needed +} + +Buffer::Error::~Error() +{ + // nothing needed +} + +Buffer::OutOfBoundsError::OutOfBoundsError() noexcept +{ + // nothing needed +} + +Buffer::OutOfBoundsError::~OutOfBoundsError() +{ + // nothing needed +} + +Buffer::InvalidOperationError::InvalidOperationError() noexcept +{ + // nothing needed +} + +Buffer::InvalidOperationError::~InvalidOperationError() +{ + // nothing needed +} + +// --- + +Buffer::Buffer() noexcept +{ + // nothing needed +} + +Buffer::~Buffer() +{ + // nothing needed +} + +void Buffer::resize(size_t newSize) +{ + throw InvalidOperationError(); +} + +uint8_t &Buffer::operator[](size_t i) +{ + if (i>=size()) throw OutOfBoundsError(); + return data()[i]; +} + +const uint8_t &Buffer::operator[](size_t i) const +{ + if (i>=size()) throw OutOfBoundsError(); + return data()[i]; +} + +uint32_t Buffer::readBE32(size_t offset) const +{ + if (OverflowCheck::sum(offset,4U)>size()) throw OutOfBoundsError(); + const uint8_t *ptr=data()+offset; + return (uint32_t(ptr[0])<<24)|(uint32_t(ptr[1])<<16)|(uint32_t(ptr[2])<<8)|uint32_t(ptr[3]); +} + +uint16_t Buffer::readBE16(size_t offset) const +{ + if (OverflowCheck::sum(offset,2U)>size()) throw OutOfBoundsError(); + const uint8_t *ptr=data()+offset; + return (uint16_t(ptr[0])<<8)|uint16_t(ptr[1]); +} + +uint64_t Buffer::readLE64(size_t offset) const +{ + if (OverflowCheck::sum(offset,8U)>size()) throw OutOfBoundsError(); + const uint8_t *ptr=data()+offset; + return (uint64_t(ptr[7])<<56)|(uint64_t(ptr[6])<<48)|(uint64_t(ptr[5])<<40)|(uint64_t(ptr[4])<<32)| + (uint64_t(ptr[3])<<24)|(uint64_t(ptr[2])<<16)|(uint64_t(ptr[1])<<8)|uint64_t(ptr[0]); +} + +uint32_t Buffer::readLE32(size_t offset) const +{ + if (OverflowCheck::sum(offset,4U)>size()) throw OutOfBoundsError(); + const uint8_t *ptr=data()+offset; + return (uint32_t(ptr[3])<<24)|(uint32_t(ptr[2])<<16)|(uint32_t(ptr[1])<<8)|uint32_t(ptr[0]); +} + +uint16_t Buffer::readLE16(size_t offset) const +{ + if (OverflowCheck::sum(offset,2U)>size()) throw OutOfBoundsError(); + const uint8_t *ptr=data()+offset; + return (uint16_t(ptr[1])<<8)|uint16_t(ptr[0]); +} + +uint8_t Buffer::read8(size_t offset) const +{ + if (offset>=size()) throw OutOfBoundsError(); + const uint8_t *ptr=reinterpret_cast<const uint8_t*>(data())+offset; + return ptr[0]; +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/Buffer.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/Buffer.hpp new file mode 100644 index 00000000..1ce5890e --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/Buffer.hpp @@ -0,0 +1,80 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef BUFFER_HPP +#define BUFFER_HPP + +#include <cstddef> +#include <cstdint> + +#include <exception> + +namespace ancient::internal +{ + +class Buffer +{ +protected: + Buffer() noexcept; + +public: + class Error : public std::exception + { + public: + Error() noexcept; + virtual ~Error(); + }; + + class OutOfBoundsError : public Error + { + public: + OutOfBoundsError() noexcept; + virtual ~OutOfBoundsError(); + }; + + class InvalidOperationError : public Error + { + public: + InvalidOperationError() noexcept; + virtual ~InvalidOperationError(); + }; + + Buffer(const Buffer&)=delete; + Buffer& operator=(const Buffer&)=delete; + + virtual ~Buffer(); + + virtual const uint8_t *data() const noexcept=0; + virtual uint8_t *data()=0; + virtual size_t size() const noexcept=0; + + template<typename T> + const T *cast() const noexcept + { + return reinterpret_cast<const T*>(data()); + } + + template<typename T> + T *cast() + { + return reinterpret_cast<T*>(data()); + } + + virtual bool isResizable() const noexcept=0; + virtual void resize(size_t newSize); + + uint8_t &operator[](size_t i); + const uint8_t &operator[](size_t i) const; + + uint32_t readBE32(size_t offset) const; + uint16_t readBE16(size_t offset) const; + + uint64_t readLE64(size_t offset) const; + uint32_t readLE32(size_t offset) const; + uint16_t readLE16(size_t offset) const; + + uint8_t read8(size_t offset) const; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/CRC16.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/CRC16.cpp new file mode 100644 index 00000000..1be46221 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/CRC16.cpp @@ -0,0 +1,44 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstdint> + +#include "Buffer.hpp" +#include "OverflowCheck.hpp" + + +namespace ancient::internal +{ + +static const uint16_t CRC16Table[256]={ + 0x0000,0xc0c1,0xc181,0x0140,0xc301,0x03c0,0x0280,0xc241,0xc601,0x06c0,0x0780,0xc741,0x0500,0xc5c1,0xc481,0x0440, + 0xcc01,0x0cc0,0x0d80,0xcd41,0x0f00,0xcfc1,0xce81,0x0e40,0x0a00,0xcac1,0xcb81,0x0b40,0xc901,0x09c0,0x0880,0xc841, + 0xd801,0x18c0,0x1980,0xd941,0x1b00,0xdbc1,0xda81,0x1a40,0x1e00,0xdec1,0xdf81,0x1f40,0xdd01,0x1dc0,0x1c80,0xdc41, + 0x1400,0xd4c1,0xd581,0x1540,0xd701,0x17c0,0x1680,0xd641,0xd201,0x12c0,0x1380,0xd341,0x1100,0xd1c1,0xd081,0x1040, + 0xf001,0x30c0,0x3180,0xf141,0x3300,0xf3c1,0xf281,0x3240,0x3600,0xf6c1,0xf781,0x3740,0xf501,0x35c0,0x3480,0xf441, + 0x3c00,0xfcc1,0xfd81,0x3d40,0xff01,0x3fc0,0x3e80,0xfe41,0xfa01,0x3ac0,0x3b80,0xfb41,0x3900,0xf9c1,0xf881,0x3840, + 0x2800,0xe8c1,0xe981,0x2940,0xeb01,0x2bc0,0x2a80,0xea41,0xee01,0x2ec0,0x2f80,0xef41,0x2d00,0xedc1,0xec81,0x2c40, + 0xe401,0x24c0,0x2580,0xe541,0x2700,0xe7c1,0xe681,0x2640,0x2200,0xe2c1,0xe381,0x2340,0xe101,0x21c0,0x2080,0xe041, + 0xa001,0x60c0,0x6180,0xa141,0x6300,0xa3c1,0xa281,0x6240,0x6600,0xa6c1,0xa781,0x6740,0xa501,0x65c0,0x6480,0xa441, + 0x6c00,0xacc1,0xad81,0x6d40,0xaf01,0x6fc0,0x6e80,0xae41,0xaa01,0x6ac0,0x6b80,0xab41,0x6900,0xa9c1,0xa881,0x6840, + 0x7800,0xb8c1,0xb981,0x7940,0xbb01,0x7bc0,0x7a80,0xba41,0xbe01,0x7ec0,0x7f80,0xbf41,0x7d00,0xbdc1,0xbc81,0x7c40, + 0xb401,0x74c0,0x7580,0xb541,0x7700,0xb7c1,0xb681,0x7640,0x7200,0xb2c1,0xb381,0x7340,0xb101,0x71c0,0x7080,0xb041, + 0x5000,0x90c1,0x9181,0x5140,0x9301,0x53c0,0x5280,0x9241,0x9601,0x56c0,0x5780,0x9741,0x5500,0x95c1,0x9481,0x5440, + 0x9c01,0x5cc0,0x5d80,0x9d41,0x5f00,0x9fc1,0x9e81,0x5e40,0x5a00,0x9ac1,0x9b81,0x5b40,0x9901,0x59c0,0x5880,0x9841, + 0x8801,0x48c0,0x4980,0x8941,0x4b00,0x8bc1,0x8a81,0x4a40,0x4e00,0x8ec1,0x8f81,0x4f40,0x8d01,0x4dc0,0x4c80,0x8c41, + 0x4400,0x84c1,0x8581,0x4540,0x8701,0x47c0,0x4680,0x8641,0x8201,0x42c0,0x4380,0x8341,0x4100,0x81c1,0x8081,0x4040}; + +uint16_t CRC16(const Buffer &buffer,size_t offset,size_t len,uint16_t accumulator) +{ + if (!len || OverflowCheck::sum(offset,len)>buffer.size()) throw Buffer::OutOfBoundsError(); + const uint8_t *ptr=buffer.data()+offset; + for (size_t i=0;i<len;i++) + accumulator=(accumulator>>8)^CRC16Table[(accumulator&0xff)^ptr[i]]; + return accumulator; +} + +uint16_t CRC16Byte(uint8_t ch,uint16_t accumulator) noexcept +{ + return (accumulator>>8)^CRC16Table[(accumulator&0xff)^ch]; +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/CRC16.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/CRC16.hpp new file mode 100644 index 00000000..8889aad2 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/CRC16.hpp @@ -0,0 +1,21 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef CRC16_HPP +#define CRC16_HPP + +#include <cstdint> + +#include "Buffer.hpp" + +namespace ancient::internal +{ + +// The most common CRC16 + +uint16_t CRC16(const Buffer &buffer,size_t offset,size_t len,uint16_t accumulator); + +uint16_t CRC16Byte(uint8_t ch,uint16_t accumulator) noexcept; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/CRC32.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/CRC32.cpp new file mode 100644 index 00000000..b5fd3ba2 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/CRC32.cpp @@ -0,0 +1,115 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstdint> + +#include "Buffer.hpp" +#include "OverflowCheck.hpp" + + +namespace ancient::internal +{ + +static const uint32_t CRC32Table[256]={ + 0x00000000U,0x77073096U,0xee0e612cU,0x990951baU,0x076dc419U,0x706af48fU,0xe963a535U,0x9e6495a3U, + 0x0edb8832U,0x79dcb8a4U,0xe0d5e91eU,0x97d2d988U,0x09b64c2bU,0x7eb17cbdU,0xe7b82d07U,0x90bf1d91U, + 0x1db71064U,0x6ab020f2U,0xf3b97148U,0x84be41deU,0x1adad47dU,0x6ddde4ebU,0xf4d4b551U,0x83d385c7U, + 0x136c9856U,0x646ba8c0U,0xfd62f97aU,0x8a65c9ecU,0x14015c4fU,0x63066cd9U,0xfa0f3d63U,0x8d080df5U, + 0x3b6e20c8U,0x4c69105eU,0xd56041e4U,0xa2677172U,0x3c03e4d1U,0x4b04d447U,0xd20d85fdU,0xa50ab56bU, + 0x35b5a8faU,0x42b2986cU,0xdbbbc9d6U,0xacbcf940U,0x32d86ce3U,0x45df5c75U,0xdcd60dcfU,0xabd13d59U, + 0x26d930acU,0x51de003aU,0xc8d75180U,0xbfd06116U,0x21b4f4b5U,0x56b3c423U,0xcfba9599U,0xb8bda50fU, + 0x2802b89eU,0x5f058808U,0xc60cd9b2U,0xb10be924U,0x2f6f7c87U,0x58684c11U,0xc1611dabU,0xb6662d3dU, + 0x76dc4190U,0x01db7106U,0x98d220bcU,0xefd5102aU,0x71b18589U,0x06b6b51fU,0x9fbfe4a5U,0xe8b8d433U, + 0x7807c9a2U,0x0f00f934U,0x9609a88eU,0xe10e9818U,0x7f6a0dbbU,0x086d3d2dU,0x91646c97U,0xe6635c01U, + 0x6b6b51f4U,0x1c6c6162U,0x856530d8U,0xf262004eU,0x6c0695edU,0x1b01a57bU,0x8208f4c1U,0xf50fc457U, + 0x65b0d9c6U,0x12b7e950U,0x8bbeb8eaU,0xfcb9887cU,0x62dd1ddfU,0x15da2d49U,0x8cd37cf3U,0xfbd44c65U, + 0x4db26158U,0x3ab551ceU,0xa3bc0074U,0xd4bb30e2U,0x4adfa541U,0x3dd895d7U,0xa4d1c46dU,0xd3d6f4fbU, + 0x4369e96aU,0x346ed9fcU,0xad678846U,0xda60b8d0U,0x44042d73U,0x33031de5U,0xaa0a4c5fU,0xdd0d7cc9U, + 0x5005713cU,0x270241aaU,0xbe0b1010U,0xc90c2086U,0x5768b525U,0x206f85b3U,0xb966d409U,0xce61e49fU, + 0x5edef90eU,0x29d9c998U,0xb0d09822U,0xc7d7a8b4U,0x59b33d17U,0x2eb40d81U,0xb7bd5c3bU,0xc0ba6cadU, + 0xedb88320U,0x9abfb3b6U,0x03b6e20cU,0x74b1d29aU,0xead54739U,0x9dd277afU,0x04db2615U,0x73dc1683U, + 0xe3630b12U,0x94643b84U,0x0d6d6a3eU,0x7a6a5aa8U,0xe40ecf0bU,0x9309ff9dU,0x0a00ae27U,0x7d079eb1U, + 0xf00f9344U,0x8708a3d2U,0x1e01f268U,0x6906c2feU,0xf762575dU,0x806567cbU,0x196c3671U,0x6e6b06e7U, + 0xfed41b76U,0x89d32be0U,0x10da7a5aU,0x67dd4accU,0xf9b9df6fU,0x8ebeeff9U,0x17b7be43U,0x60b08ed5U, + 0xd6d6a3e8U,0xa1d1937eU,0x38d8c2c4U,0x4fdff252U,0xd1bb67f1U,0xa6bc5767U,0x3fb506ddU,0x48b2364bU, + 0xd80d2bdaU,0xaf0a1b4cU,0x36034af6U,0x41047a60U,0xdf60efc3U,0xa867df55U,0x316e8eefU,0x4669be79U, + 0xcb61b38cU,0xbc66831aU,0x256fd2a0U,0x5268e236U,0xcc0c7795U,0xbb0b4703U,0x220216b9U,0x5505262fU, + 0xc5ba3bbeU,0xb2bd0b28U,0x2bb45a92U,0x5cb36a04U,0xc2d7ffa7U,0xb5d0cf31U,0x2cd99e8bU,0x5bdeae1dU, + 0x9b64c2b0U,0xec63f226U,0x756aa39cU,0x026d930aU,0x9c0906a9U,0xeb0e363fU,0x72076785U,0x05005713U, + 0x95bf4a82U,0xe2b87a14U,0x7bb12baeU,0x0cb61b38U,0x92d28e9bU,0xe5d5be0dU,0x7cdcefb7U,0x0bdbdf21U, + 0x86d3d2d4U,0xf1d4e242U,0x68ddb3f8U,0x1fda836eU,0x81be16cdU,0xf6b9265bU,0x6fb077e1U,0x18b74777U, + 0x88085ae6U,0xff0f6a70U,0x66063bcaU,0x11010b5cU,0x8f659effU,0xf862ae69U,0x616bffd3U,0x166ccf45U, + 0xa00ae278U,0xd70dd2eeU,0x4e048354U,0x3903b3c2U,0xa7672661U,0xd06016f7U,0x4969474dU,0x3e6e77dbU, + 0xaed16a4aU,0xd9d65adcU,0x40df0b66U,0x37d83bf0U,0xa9bcae53U,0xdebb9ec5U,0x47b2cf7fU,0x30b5ffe9U, + 0xbdbdf21cU,0xcabac28aU,0x53b39330U,0x24b4a3a6U,0xbad03605U,0xcdd70693U,0x54de5729U,0x23d967bfU, + 0xb3667a2eU,0xc4614ab8U,0x5d681b02U,0x2a6f2b94U,0xb40bbe37U,0xc30c8ea1U,0x5a05df1bU,0x2d02ef8dU}; + +uint32_t CRC32(const Buffer &buffer,size_t offset,size_t len,uint32_t accumulator) +{ + + if (!len || OverflowCheck::sum(offset,len)>buffer.size()) throw Buffer::OutOfBoundsError(); + const uint8_t *ptr=buffer.data()+offset; + accumulator=~accumulator; + for (size_t i=0;i<len;i++) + accumulator=(accumulator>>8)^CRC32Table[(accumulator&0xff)^ptr[i]]; + return ~accumulator; +} + +uint32_t CRC32Byte(uint8_t ch,uint32_t accumulator) noexcept +{ + return ~((~accumulator>>8)^CRC32Table[(~accumulator&0xff)^ch]); +} + +// instead of bit-twiddling lets have a separate implementation for reverse + +// same table as the previous one, but reflected +static const uint32_t CRC32RevTable[256]={ + 0x00000000U,0x04c11db7U,0x09823b6eU,0x0d4326d9U,0x130476dcU,0x17c56b6bU,0x1a864db2U,0x1e475005U, + 0x2608edb8U,0x22c9f00fU,0x2f8ad6d6U,0x2b4bcb61U,0x350c9b64U,0x31cd86d3U,0x3c8ea00aU,0x384fbdbdU, + 0x4c11db70U,0x48d0c6c7U,0x4593e01eU,0x4152fda9U,0x5f15adacU,0x5bd4b01bU,0x569796c2U,0x52568b75U, + 0x6a1936c8U,0x6ed82b7fU,0x639b0da6U,0x675a1011U,0x791d4014U,0x7ddc5da3U,0x709f7b7aU,0x745e66cdU, + 0x9823b6e0U,0x9ce2ab57U,0x91a18d8eU,0x95609039U,0x8b27c03cU,0x8fe6dd8bU,0x82a5fb52U,0x8664e6e5U, + 0xbe2b5b58U,0xbaea46efU,0xb7a96036U,0xb3687d81U,0xad2f2d84U,0xa9ee3033U,0xa4ad16eaU,0xa06c0b5dU, + 0xd4326d90U,0xd0f37027U,0xddb056feU,0xd9714b49U,0xc7361b4cU,0xc3f706fbU,0xceb42022U,0xca753d95U, + 0xf23a8028U,0xf6fb9d9fU,0xfbb8bb46U,0xff79a6f1U,0xe13ef6f4U,0xe5ffeb43U,0xe8bccd9aU,0xec7dd02dU, + 0x34867077U,0x30476dc0U,0x3d044b19U,0x39c556aeU,0x278206abU,0x23431b1cU,0x2e003dc5U,0x2ac12072U, + 0x128e9dcfU,0x164f8078U,0x1b0ca6a1U,0x1fcdbb16U,0x018aeb13U,0x054bf6a4U,0x0808d07dU,0x0cc9cdcaU, + 0x7897ab07U,0x7c56b6b0U,0x71159069U,0x75d48ddeU,0x6b93dddbU,0x6f52c06cU,0x6211e6b5U,0x66d0fb02U, + 0x5e9f46bfU,0x5a5e5b08U,0x571d7dd1U,0x53dc6066U,0x4d9b3063U,0x495a2dd4U,0x44190b0dU,0x40d816baU, + 0xaca5c697U,0xa864db20U,0xa527fdf9U,0xa1e6e04eU,0xbfa1b04bU,0xbb60adfcU,0xb6238b25U,0xb2e29692U, + 0x8aad2b2fU,0x8e6c3698U,0x832f1041U,0x87ee0df6U,0x99a95df3U,0x9d684044U,0x902b669dU,0x94ea7b2aU, + 0xe0b41de7U,0xe4750050U,0xe9362689U,0xedf73b3eU,0xf3b06b3bU,0xf771768cU,0xfa325055U,0xfef34de2U, + 0xc6bcf05fU,0xc27dede8U,0xcf3ecb31U,0xcbffd686U,0xd5b88683U,0xd1799b34U,0xdc3abdedU,0xd8fba05aU, + 0x690ce0eeU,0x6dcdfd59U,0x608edb80U,0x644fc637U,0x7a089632U,0x7ec98b85U,0x738aad5cU,0x774bb0ebU, + 0x4f040d56U,0x4bc510e1U,0x46863638U,0x42472b8fU,0x5c007b8aU,0x58c1663dU,0x558240e4U,0x51435d53U, + 0x251d3b9eU,0x21dc2629U,0x2c9f00f0U,0x285e1d47U,0x36194d42U,0x32d850f5U,0x3f9b762cU,0x3b5a6b9bU, + 0x0315d626U,0x07d4cb91U,0x0a97ed48U,0x0e56f0ffU,0x1011a0faU,0x14d0bd4dU,0x19939b94U,0x1d528623U, + 0xf12f560eU,0xf5ee4bb9U,0xf8ad6d60U,0xfc6c70d7U,0xe22b20d2U,0xe6ea3d65U,0xeba91bbcU,0xef68060bU, + 0xd727bbb6U,0xd3e6a601U,0xdea580d8U,0xda649d6fU,0xc423cd6aU,0xc0e2d0ddU,0xcda1f604U,0xc960ebb3U, + 0xbd3e8d7eU,0xb9ff90c9U,0xb4bcb610U,0xb07daba7U,0xae3afba2U,0xaafbe615U,0xa7b8c0ccU,0xa379dd7bU, + 0x9b3660c6U,0x9ff77d71U,0x92b45ba8U,0x9675461fU,0x8832161aU,0x8cf30badU,0x81b02d74U,0x857130c3U, + 0x5d8a9099U,0x594b8d2eU,0x5408abf7U,0x50c9b640U,0x4e8ee645U,0x4a4ffbf2U,0x470cdd2bU,0x43cdc09cU, + 0x7b827d21U,0x7f436096U,0x7200464fU,0x76c15bf8U,0x68860bfdU,0x6c47164aU,0x61043093U,0x65c52d24U, + 0x119b4be9U,0x155a565eU,0x18197087U,0x1cd86d30U,0x029f3d35U,0x065e2082U,0x0b1d065bU,0x0fdc1becU, + 0x3793a651U,0x3352bbe6U,0x3e119d3fU,0x3ad08088U,0x2497d08dU,0x2056cd3aU,0x2d15ebe3U,0x29d4f654U, + 0xc5a92679U,0xc1683bceU,0xcc2b1d17U,0xc8ea00a0U,0xd6ad50a5U,0xd26c4d12U,0xdf2f6bcbU,0xdbee767cU, + 0xe3a1cbc1U,0xe760d676U,0xea23f0afU,0xeee2ed18U,0xf0a5bd1dU,0xf464a0aaU,0xf9278673U,0xfde69bc4U, + 0x89b8fd09U,0x8d79e0beU,0x803ac667U,0x84fbdbd0U,0x9abc8bd5U,0x9e7d9662U,0x933eb0bbU,0x97ffad0cU, + 0xafb010b1U,0xab710d06U,0xa6322bdfU,0xa2f33668U,0xbcb4666dU,0xb8757bdaU,0xb5365d03U,0xb1f740b4U}; + +uint32_t CRC32Rev(const Buffer &buffer,size_t offset,size_t len,uint32_t accumulator) +{ + + if (!len || OverflowCheck::sum(offset,len)>buffer.size()) throw Buffer::OutOfBoundsError(); + const uint8_t *ptr=buffer.data()+offset; + accumulator=~accumulator; + for (size_t i=0;i<len;i++) + accumulator=(accumulator<<8)^CRC32RevTable[(accumulator>>24)^ptr[i]]; + return ~accumulator; +} + +uint32_t CRC32RevByte(uint8_t ch,uint32_t &accumulator) noexcept +{ + return ~((~accumulator<<8)^CRC32RevTable[(~accumulator>>24)^ch]); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/CRC32.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/CRC32.hpp new file mode 100644 index 00000000..540de2d4 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/CRC32.hpp @@ -0,0 +1,27 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef CRC32_HPP +#define CRC32_HPP + +#include <cstdint> + +#include "Buffer.hpp" + +namespace ancient::internal +{ + +// The most common CRC32 + +uint32_t CRC32(const Buffer &buffer,size_t offset,size_t len,uint32_t accumulator); + +uint32_t CRC32Byte(uint8_t ch,uint32_t accumulator) noexcept; + +// Same polynomial, but in reverse... + +uint32_t CRC32Rev(const Buffer &buffer,size_t offset,size_t len,uint32_t accumulator); + +uint32_t CRC32RevByte(uint8_t ch,uint32_t accumulator) noexcept; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/Common.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/Common.cpp new file mode 100644 index 00000000..380abc3a --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/Common.cpp @@ -0,0 +1,28 @@ +/* Copyright (C) Teemu Suutari */ + +#include "Common.hpp" + + +namespace ancient::internal +{ + +uint32_t rotateBits(uint32_t value,uint32_t count) +{ + static const uint8_t rotateNibble[16]={ + 0x0,0x8,0x4,0xc, + 0x2,0xa,0x6,0xe, + 0x1,0x9,0x5,0xd, + 0x3,0xb,0x7,0xf + }; + + uint32_t ret=0; + for (uint32_t i=0;i<count;i+=4) + { + ret=(ret<<4)|rotateNibble[value&0xf]; + value>>=4; + } + ret>>=(4-count)&3; + return ret; +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/Common.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/Common.hpp new file mode 100644 index 00000000..5dab2696 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/Common.hpp @@ -0,0 +1,46 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef COMMON_HPP +#define COMMON_HPP + +#include <cstdint> + +#include <string> +#include <vector> + +namespace ancient::internal +{ + +constexpr uint16_t MultiChar2(const char (&cc)[3]) noexcept +{ + return static_cast<uint16_t>((static_cast<uint8_t>(cc[0]) << 8) | static_cast<uint8_t>(cc[1])); +} + +constexpr uint32_t FourCC(const char (&cc)[5]) noexcept +{ + return static_cast<uint32_t>((static_cast<uint8_t>(cc[0]) << 24) | (static_cast<uint8_t>(cc[1]) << 16) | (static_cast<uint8_t>(cc[2]) << 8) | static_cast<uint8_t>(cc[3])); +} + +constexpr bool isValidSize(uint64_t &value) noexcept +{ +#if INTPTR_MAX == INT32_MAX + return value<0x1'0000'0000ULL; +#else + return true; +#endif +} + +constexpr bool isValidSize(off_t &value) noexcept +{ +#if INTPTR_MAX == INT32_MAX + return value<0x1'0000'0000ULL; +#else + return true; +#endif +} + +uint32_t rotateBits(uint32_t value,uint32_t count); + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/MemoryBuffer.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/MemoryBuffer.cpp new file mode 100644 index 00000000..fa0dcef8 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/MemoryBuffer.cpp @@ -0,0 +1,70 @@ +/* Copyright (C) Teemu Suutari */ + +#include <cstring> +#include <cstdlib> + +#include <memory> +#include <new> + +#include "MemoryBuffer.hpp" +#include "OverflowCheck.hpp" + + +namespace ancient::internal +{ + +MemoryBuffer::MemoryBuffer(size_t size) : + _data(reinterpret_cast<uint8_t*>(std::malloc(size))), + _size(size) +{ + if (!_data) throw std::bad_alloc(); +} + +MemoryBuffer::MemoryBuffer(const Buffer &src,size_t offset,size_t size) : + MemoryBuffer(size) +{ + if(OverflowCheck::sum(offset,size)>src.size()) throw InvalidOperationError(); + std::memcpy(_data,src.data()+offset,size); +} + + +MemoryBuffer::~MemoryBuffer() +{ + std::free(_data); +} + +const uint8_t *MemoryBuffer::data() const noexcept +{ + return _data; +} + +uint8_t *MemoryBuffer::data() +{ + return _data; +} + +size_t MemoryBuffer::size() const noexcept +{ + return _size; +} + +bool MemoryBuffer::isResizable() const noexcept +{ + return true; +} + +void MemoryBuffer::resize(size_t newSize) +{ + uint8_t *newData=reinterpret_cast<uint8_t*>(std::realloc(_data,newSize)); + if (!newData) + { + std::free(_data); + _data=nullptr; + _size=0; + throw std::bad_alloc(); + } + _data=newData; + _size=newSize; +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/MemoryBuffer.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/MemoryBuffer.hpp new file mode 100644 index 00000000..d19215a6 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/MemoryBuffer.hpp @@ -0,0 +1,34 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef MEMORYBUFFER_HPP +#define MEMORYBUFFER_HPP + +#include <memory> + +#include "Buffer.hpp" + +namespace ancient::internal +{ + +class MemoryBuffer : public Buffer +{ +public: + MemoryBuffer(size_t size); + MemoryBuffer(const Buffer &src,size_t offset,size_t size); + virtual ~MemoryBuffer() override final; + + virtual const uint8_t *data() const noexcept override final; + virtual uint8_t *data() override final; + virtual size_t size() const noexcept override final; + + virtual bool isResizable() const noexcept override final; + virtual void resize(size_t newSize) override final; + +private: + uint8_t* _data; + size_t _size; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/OverflowCheck.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/OverflowCheck.hpp new file mode 100644 index 00000000..524dd2f4 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/OverflowCheck.hpp @@ -0,0 +1,32 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef OVERFLOW_CHECK_HPP +#define OVERFLOW_CHECK_HPP + +#include "Buffer.hpp" + +namespace ancient::internal +{ + +class OverflowCheck +{ +public: + template<typename T,typename U> + static T sum(T a,U b) + { + // TODO: Add type traits to handle signed integers + T ret=a+b; + if (ret<a) throw Buffer::OutOfBoundsError(); + return ret; + } + + template<typename T,typename U,typename ...Args> + static T sum(T a,U b,Args... args) + { + return sum(sum(a,b),args...); + } +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/StaticBuffer.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/StaticBuffer.cpp new file mode 100644 index 00000000..16590752 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/StaticBuffer.cpp @@ -0,0 +1,41 @@ +/* Copyright (C) Teemu Suutari */ + +#include "StaticBuffer.hpp" + + +namespace ancient::internal +{ + +ConstStaticBuffer::ConstStaticBuffer(const uint8_t *data,size_t length) : + _data(data), + _length(length) +{ + // nothing needed +} + +ConstStaticBuffer::~ConstStaticBuffer() +{ + // nothing needed +} + +const uint8_t *ConstStaticBuffer::data() const noexcept +{ + return _data; +} + +uint8_t *ConstStaticBuffer::data() +{ + throw InvalidOperationError(); +} + +size_t ConstStaticBuffer::size() const noexcept +{ + return _length; +} + +bool ConstStaticBuffer::isResizable() const noexcept +{ + return false; +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/StaticBuffer.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/StaticBuffer.hpp new file mode 100644 index 00000000..2fcff561 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/StaticBuffer.hpp @@ -0,0 +1,72 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef STATICBUFFER_HPP +#define STATICBUFFER_HPP + +#include <cstddef> +#include <cstdint> + +#include "Buffer.hpp" + +namespace ancient::internal +{ + +template<size_t N> +class StaticBuffer : public Buffer +{ +public: + StaticBuffer(const StaticBuffer&)=delete; + StaticBuffer& operator=(const StaticBuffer&)=delete; + + StaticBuffer() { } + + virtual ~StaticBuffer() { } + + virtual const uint8_t *data() const noexcept override + { + return _data; + } + + virtual uint8_t *data() override + { + return _data; + } + + virtual size_t size() const noexcept override + { + return N; + } + + virtual bool isResizable() const noexcept override + { + return false; + } + +private: + uint8_t _data[N]; +}; + + +class ConstStaticBuffer : public Buffer +{ +public: + ConstStaticBuffer(const ConstStaticBuffer&)=delete; + ConstStaticBuffer& operator=(const ConstStaticBuffer&)=delete; + + ConstStaticBuffer(const uint8_t *data,size_t length); + virtual ~ConstStaticBuffer(); + + virtual const uint8_t *data() const noexcept override; + virtual uint8_t *data() override; + + virtual size_t size() const noexcept override; + virtual bool isResizable() const noexcept override; + +private: + const uint8_t *_data; + size_t _length; +}; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/SubBuffer.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/SubBuffer.cpp new file mode 100644 index 00000000..8864667a --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/SubBuffer.cpp @@ -0,0 +1,21 @@ +/* Copyright (C) Teemu Suutari */ + +#include "SubBuffer.hpp" + + +namespace ancient::internal +{ + +template <> +uint8_t *GenericSubBuffer<Buffer>::data() +{ + return _base.data()+_start; +} + +template <> +uint8_t *GenericSubBuffer<const Buffer>::data() +{ + throw InvalidOperationError(); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/SubBuffer.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/SubBuffer.hpp new file mode 100644 index 00000000..3c76a8cd --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/SubBuffer.hpp @@ -0,0 +1,75 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef SUBBUFFER_HPP +#define SUBBUFFER_HPP + +#include <cstddef> +#include <cstdint> + +#include "Buffer.hpp" +#include "OverflowCheck.hpp" + +namespace ancient::internal +{ + +// helpers to splice Buffer + +template <typename T> +class GenericSubBuffer : public Buffer +{ +public: + GenericSubBuffer(const GenericSubBuffer&)=delete; + GenericSubBuffer& operator=(const GenericSubBuffer&)=delete; + + GenericSubBuffer(T &base,size_t start,size_t length) : + _base(base), + _start(start), + _length(length) + { + // if the sub-buffer is invalid, we set both _start and _length to 0 + if (OverflowCheck::sum(start,length)>_base.size()) + { + _start=0; + _length=0; + } + } + + virtual ~GenericSubBuffer() { } + + virtual const uint8_t *data() const noexcept override + { + return _base.data()+_start; + } + + virtual uint8_t *data() override; + + virtual size_t size() const noexcept override + { + return _length; + } + + virtual bool isResizable() const noexcept override + { + return false; + } + + // can only make the buffer smaller, can't run away from the current bounds + void adjust(size_t start,size_t length) + { + if (start<_start || OverflowCheck::sum(start,length)>_start+_length) throw OutOfBoundsError(); + _start=start; + _length=length; + } + +private: + T &_base; + size_t _start; + size_t _length; +}; + +typedef GenericSubBuffer<Buffer> SubBuffer; +typedef GenericSubBuffer<const Buffer> ConstSubBuffer; + +} + +#endif diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/WrappedVectorBuffer.cpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/WrappedVectorBuffer.cpp new file mode 100644 index 00000000..aaf7579e --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/WrappedVectorBuffer.cpp @@ -0,0 +1,45 @@ +/* Copyright (C) Teemu Suutari */ + +#include "WrappedVectorBuffer.hpp" + + +namespace ancient::internal +{ + +WrappedVectorBuffer::WrappedVectorBuffer(std::vector<uint8_t> &refdata) : + _refdata(refdata) +{ + return; +} + +WrappedVectorBuffer::~WrappedVectorBuffer() +{ + return; +} + +const uint8_t *WrappedVectorBuffer::data() const noexcept +{ + return _refdata.data(); +} + +uint8_t *WrappedVectorBuffer::data() +{ + return _refdata.data(); +} + +size_t WrappedVectorBuffer::size() const noexcept +{ + return _refdata.size(); +} + +bool WrappedVectorBuffer::isResizable() const noexcept +{ + return true; +} + +void WrappedVectorBuffer::resize(size_t newSize) +{ + _refdata.resize(newSize); +} + +} diff --git a/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/WrappedVectorBuffer.hpp b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/WrappedVectorBuffer.hpp new file mode 100644 index 00000000..c3a4a921 --- /dev/null +++ b/Src/external_dependencies/openmpt-trunk/include/ancient/src/common/WrappedVectorBuffer.hpp @@ -0,0 +1,35 @@ +/* Copyright (C) Teemu Suutari */ + +#ifndef WRAPPEDVECTORBUFFER_HPP +#define WRAPPEDVECTORBUFFER_HPP + +#include <vector> + +#include <cstddef> +#include <cstdint> + +#include "Buffer.hpp" + +namespace ancient::internal +{ + +class WrappedVectorBuffer : public Buffer +{ +public: + WrappedVectorBuffer(std::vector<uint8_t> &refdata); + virtual ~WrappedVectorBuffer() override final; + + virtual const uint8_t *data() const noexcept override final; + virtual uint8_t *data() override final; + virtual size_t size() const noexcept override final; + + virtual bool isResizable() const noexcept override final; + virtual void resize(size_t newSize) override final; + +private: + std::vector<uint8_t> & _refdata; +}; + +} + +#endif |