StreamCompression.h

Go to the documentation of this file.

//
// Copyright (c) 2012-2018 DreamWorks Animation LLC
//
// All rights reserved. This software is distributed under the
// Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )
//
// Redistributions of source code must retain the above copyright
// and license notice and the following restrictions and disclaimer.
//
// *     Neither the name of DreamWorks Animation nor the names of
// its contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// IN NO EVENT SHALL THE COPYRIGHT HOLDERS' AND CONTRIBUTORS' AGGREGATE
// LIABILITY FOR ALL CLAIMS REGARDLESS OF THEIR BASIS EXCEED US$250.00.
//
 
 
#ifndef OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
#define OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
 
#include <openvdb/io/io.h>
#include <tbb/spin_mutex.h>
#include <memory>
#include <string>
 
 
class TestStreamCompression;
 
namespace openvdb {
OPENVDB_USE_VERSION_NAMESPACE
namespace OPENVDB_VERSION_NAME {
namespace compression {
 
 
// This is the minimum number of bytes below which Blosc compression is not used to
// avoid unecessary computation, as Blosc offers minimal compression until this limit
static const int BLOSC_MINIMUM_BYTES = 48;
 
// This is the minimum number of bytes below which the array is padded with zeros up
// to this number of bytes to allow Blosc to perform compression with small arrays
static const int BLOSC_PAD_BYTES = 128;
 
 
OPENVDB_API bool bloscCanCompress();
 
OPENVDB_API size_t bloscUncompressedSize(const char* buffer);
 
OPENVDB_API void bloscCompress(char* compressedBuffer, size_t& compressedBytes,
    const size_t bufferBytes, const char* uncompressedBuffer, const size_t uncompressedBytes);
 
OPENVDB_API std::unique_ptr<char[]> bloscCompress(const char* buffer,
    const size_t uncompressedBytes, size_t& compressedBytes, const bool resize = true);
 
OPENVDB_API size_t bloscCompressedSize(const char* buffer, const size_t uncompressedBytes);
 
OPENVDB_API void bloscDecompress(char* uncompressedBuffer, const size_t expectedBytes,
    const size_t bufferBytes, const char* compressedBuffer);
 
OPENVDB_API std::unique_ptr<char[]> bloscDecompress(const char* buffer,
    const size_t expectedBytes, const bool resize = true);
 
 
 
 
// 1MB = 1048576 Bytes
static const int PageSize = 1024 * 1024;
 
 
class OPENVDB_API Page
{
private:
    struct Info
    {
        io::MappedFile::Ptr mappedFile;
        SharedPtr<io::StreamMetadata> meta;
        std::streamoff filepos;
        long compressedBytes;
        long uncompressedBytes;
    }; // Info
 
public:
    using Ptr = std::shared_ptr<Page>;
 
    Page() = default;
 
    void load() const;
 
    long uncompressedBytes() const;
 
    const char* buffer(const int index) const;
 
    void readHeader(std::istream&);
 
    void readBuffers(std::istream&, bool delayed);
 
    bool isOutOfCore() const;
 
private:
    void copy(const std::unique_ptr<char[]>& temp, int pageSize);
 
    void decompress(const std::unique_ptr<char[]>& temp);
 
    void doLoad() const;
 
    std::unique_ptr<Info> mInfo = std::unique_ptr<Info>(new Info);
    std::unique_ptr<char[]> mData;
    tbb::spin_mutex mMutex;
}; // class Page
 
 
class OPENVDB_API PageHandle
{
public:
    using Ptr = std::shared_ptr<PageHandle>;
 
    PageHandle(const Page::Ptr& page, const int index, const int size);
 
    Page& page();
 
    std::unique_ptr<char[]> read();
 
protected:
    friend class ::TestStreamCompression;
 
private:
    Page::Ptr mPage;
    int mIndex = -1;
    int mSize = 0;
}; // class PageHandle
 
 
class OPENVDB_API PagedInputStream
{
public:
    using Ptr = std::shared_ptr<PagedInputStream>;
 
    PagedInputStream() = default;
 
    explicit PagedInputStream(std::istream& is);
 
    void setSizeOnly(bool sizeOnly) { mSizeOnly = sizeOnly; }
    bool sizeOnly() const { return mSizeOnly; }
 
    // @brief Set and get the input stream
    std::istream& getInputStream() { assert(mIs); return *mIs; }
    void setInputStream(std::istream& is) { mIs = &is; }
 
    PageHandle::Ptr createHandle(std::streamsize n);
 
    void read(PageHandle::Ptr& pageHandle, std::streamsize n, bool delayed = true);
 
private:
    int mByteIndex = 0;
    int mUncompressedBytes = 0;
    std::istream* mIs = nullptr;
    Page::Ptr mPage;
    bool mSizeOnly = false;
}; // class PagedInputStream
 
 
class OPENVDB_API PagedOutputStream
{
public:
    using Ptr = std::shared_ptr<PagedOutputStream>;
 
    PagedOutputStream();
 
    explicit PagedOutputStream(std::ostream& os);
 
    void setSizeOnly(bool sizeOnly) { mSizeOnly = sizeOnly; }
    bool sizeOnly() const { return mSizeOnly; }
 
    std::ostream& getOutputStream() { assert(mOs); return *mOs; }
    void setOutputStream(std::ostream& os) { mOs = &os; }
 
    PagedOutputStream& write(const char* str, std::streamsize n);
 
    void flush();
 
private:
    void compressAndWrite(const char* buffer, size_t size);
 
    void resize(size_t size);
 
    std::unique_ptr<char[]> mData = std::unique_ptr<char[]>(new char[PageSize]);
    std::unique_ptr<char[]> mCompressedData = nullptr;
    size_t mCapacity = PageSize;
    int mBytes = 0;
    std::ostream* mOs = nullptr;
    bool mSizeOnly = false;
}; // class PagedOutputStream
 
 
} // namespace compression
} // namespace OPENVDB_VERSION_NAME
} // namespace openvdb
 
#endif // OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
 
// Copyright (c) 2012-2018 DreamWorks Animation LLC
// All rights reserved. This software is distributed under the
// Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )