synchronized_heap_opencl.h

#pragma once

#ifndef SYNCHRONIZED_HEAP_OPENCL_H
#define SYNCHRONIZED_HEAP_OPENCL_H




#include "settings.h"




#if ( SETTING_OPENCL != 0 )




#include "synchronized_heap_base.h"
#include "platform.h"
#include "opencl.h"
#include "exception.h"

#include <boost/bimap.hpp>
#include <boost/bimap/multiset_of.hpp>
#include <boost/bimap/set_of.hpp>

#include <unordered_set>
#include <memory>

#include <CL/cl.h>




namespace Synchronized {




/*==============================================================================
    Heap< PLATFORM_OPENCL > specialization
==============================================================================*/


template<>
struct Heap< PLATFORM_OPENCL > {

    template< typename t_Type >
    struct Pointer;

    typedef cl_mem OpenCLType;


    inline Heap( std::shared_ptr< OpenCL::State > const& pState, size_t const size );


    template< typename t_Type >
    inline Pointer< t_Type > Allocate();

    template< typename t_Type >
    inline Pointer< t_Type[] > Allocate( size_t const count );


    inline void Synchronize();


    inline OpenCLType OpenCLConvert() const;


private:

    struct ObjectBase {

        inline ObjectBase( Heap* const pParent, std::pair< size_t, size_t > const& span );

        virtual ~ObjectBase() = 0;


        virtual void OpenCLWrite( void* const destination ) const = 0;


        inline unsigned int References() const;

        inline void AddReference();
        inline void ReleaseReference();


        inline std::pair< size_t, size_t > const& Span() const;

        inline void Mark() const;


    private:

        unsigned int m_refcount;

        Heap* m_pParent;
        std::pair< size_t, size_t > m_span;
    };

    template< typename t_Type >
    struct Object : public ObjectBase {

        typedef t_Type Type;


        inline Object( Heap* const pParent, std::pair< size_t, size_t > const& span );

        virtual ~Object();


        virtual void OpenCLWrite( void* const destination ) const;


        inline t_Type const* Get() const;
        inline void Set( t_Type const& value );


    private:

        t_Type m_data;
    };

    template< typename t_Type >
    struct Object< t_Type[] > : public ObjectBase {

        typedef t_Type Type;


        inline Object( Heap* const pParent, std::pair< size_t, size_t > const& span, size_t const count );

        virtual ~Object();


        virtual void OpenCLWrite( void* const destination ) const;


        inline t_Type const* Get( size_t const index = 0 ) const;
        inline void Set( size_t const index, t_Type const& value );


    private:

        size_t m_count;
        t_Type* m_array;
    };


public:

    template< typename t_Type >
    struct Pointer {

        typedef typename Object< t_Type >::Type const element_type;
        typedef cl_uint OpenCLType;


        inline Pointer();
        inline Pointer( Pointer const& other );

        inline ~Pointer();


        inline void reset();
        inline void swap( Pointer& other );

        inline unsigned int use_count() const;
        inline bool unique() const;


        inline element_type* get() const;

        inline void Assign( element_type const& data ) const;
        inline void Assign( size_t const index, element_type const& data ) const;


        inline Pointer const& operator=( Pointer const& other );


        inline operator bool() const;

        inline element_type* operator->() const;
        inline element_type& operator*() const;

        inline element_type& operator[]( size_t const index ) const;

        inline bool operator==( Pointer const& other ) const;
        inline bool operator!=( Pointer const& other ) const;


        inline OpenCLType OpenCLConvert() const;


    private:

        inline Pointer( Object< t_Type >* pointer );


        Object< t_Type >* m_pointer;


        friend struct Heap< PLATFORM_OPENCL >;
    };


private:

    struct ObjectComparison {

        inline bool operator()( ObjectBase const* const pLeft, ObjectBase const* const pRight ) const;
    };


    template< typename t_Type >
    inline std::pair< size_t, size_t > AllocateSpan( size_t const count = 1 );

    inline void FreeObject( ObjectBase const* pObject );

    inline void MarkObject( ObjectBase const* pObject );


    typedef boost::bimap< boost::bimaps::set_of< size_t >, boost::bimaps::multiset_of< size_t > > FreeSpans;
    typedef std::unordered_set< ObjectBase const* > MarkedObjects;


    std::shared_ptr< OpenCL::State > m_pState;
    std::unique_ptr< _cl_mem, OpenCL::Deleter< _cl_mem > > m_buffer;

    FreeSpans m_freeSpans;
    MarkedObjects m_markedObjects;
};




/*==============================================================================
    Heap< PLATFORM_OPENCL >::ObjectBase methods
==============================================================================*/


Heap< PLATFORM_OPENCL >::ObjectBase::ObjectBase( Heap* const pParent, std::pair< size_t, size_t > const& span ) :
    m_refcount( 0 ),
    m_pParent( pParent ),
    m_span( span )
{
}


unsigned int Heap< PLATFORM_OPENCL >::ObjectBase::References() const {

    return m_refcount;
}


void Heap< PLATFORM_OPENCL >::ObjectBase::AddReference() {

    ++m_refcount;
}


void Heap< PLATFORM_OPENCL >::ObjectBase::ReleaseReference() {

    assert( m_refcount > 0 );
    if ( --m_refcount == 0 )
        delete this;
}


std::pair< size_t, size_t > const& Heap< PLATFORM_OPENCL >::ObjectBase::Span() const {

    return m_span;
}


void Heap< PLATFORM_OPENCL >::ObjectBase::Mark() const {

    assert( m_pParent != NULL );
    m_pParent->MarkObject( this );
}




/*==============================================================================
    Heap< PLATFORM_OPENCL >::Object methods
==============================================================================*/


template< typename t_Type >
Heap< PLATFORM_OPENCL >::Object< t_Type >::Object( Heap* const pParent, std::pair< size_t, size_t > const& span ) :
    ObjectBase( pParent, span )
{
}


template< typename t_Type >
Heap< PLATFORM_OPENCL >::Object< t_Type >::~Object() {
}


template< typename t_Type >
void Heap< PLATFORM_OPENCL >::Object< t_Type >::OpenCLWrite( void* const destination ) const {

    typedef typename OpenCL::Converter< t_Type >::Type OpenCLType;
    *reinterpret_cast< OpenCLType* >( destination ) = OpenCL::Converter< t_Type >::Convert( m_data );
}


template< typename t_Type >
t_Type const* Heap< PLATFORM_OPENCL >::Object< t_Type >::Get() const {

    return &m_data;
}


template< typename t_Type >
void Heap< PLATFORM_OPENCL >::Object< t_Type >::Set( t_Type const& data ) {

    m_data = data;
    Mark();
}


template< typename t_Type >
Heap< PLATFORM_OPENCL >::Object< t_Type[] >::Object( Heap* const pParent, std::pair< size_t, size_t > const& span, size_t const count ) :
    ObjectBase( pParent, span ),
    m_count( count ),
    m_array( new t_Type[ m_count ] )
{
}


template< typename t_Type >
Heap< PLATFORM_OPENCL >::Object< t_Type[] >::~Object() {

    delete[] m_array;
}


template< typename t_Type >
void Heap< PLATFORM_OPENCL >::Object< t_Type[] >::OpenCLWrite( void* const destination ) const {

    typedef typename OpenCL::Converter< t_Type >::Type OpenCLType;

    OpenCLType* ii    = reinterpret_cast< OpenCLType* >( destination );
    OpenCLType* iiEnd = ii + m_count;
    t_Type const* jj = m_array;
    for ( ; ii != iiEnd; ++ii, ++jj )
        *ii = OpenCL::Converter< t_Type >::Convert( *jj );
}


template< typename t_Type >
t_Type const* Heap< PLATFORM_OPENCL >::Object< t_Type[] >::Get( size_t const index ) const {

    assert( index < m_count );
    return( m_array + index );
}


template< typename t_Type >
void Heap< PLATFORM_OPENCL >::Object< t_Type[] >::Set( size_t const index, t_Type const& data ) {

    assert( index < m_count );
    m_array[ index ] = data;
    Mark();
}




/*==============================================================================
    Heap< PLATFORM_OPENCL >::Pointer methods
==============================================================================*/


template< typename t_Type >
Heap< PLATFORM_OPENCL >::Pointer< t_Type >::Pointer() : m_pointer( NULL ) {
}


template< typename t_Type >
Heap< PLATFORM_OPENCL >::Pointer< t_Type >::Pointer( Pointer const& other ) : m_pointer( other.m_pointer ) {

    if ( m_pointer != NULL )
        m_pointer->AddReference();
}


template< typename t_Type >
Heap< PLATFORM_OPENCL >::Pointer< t_Type >::~Pointer() {

    reset();
}


template< typename t_Type >
void Heap< PLATFORM_OPENCL >::Pointer< t_Type >::reset() {

    if ( m_pointer != NULL ) {

        m_pointer->ReleaseReference();
        m_pointer = NULL;
    }
}


template< typename t_Type >
void Heap< PLATFORM_OPENCL >::Pointer< t_Type >::swap( Pointer& other ) {

    std::swap( m_pointer, other.m_pointer );
}


template< typename t_Type >
unsigned int Heap< PLATFORM_OPENCL >::Pointer< t_Type >::use_count() const {

    unsigned int result = 0;
    if ( m_pointer != NULL )
        result = m_pointer->References();
    return result;
}


template< typename t_Type >
bool Heap< PLATFORM_OPENCL >::Pointer< t_Type >::unique() const {

    return( use_count() == 1 );
}


template< typename t_Type >
typename Heap< PLATFORM_OPENCL >::Object< t_Type >::Type const* Heap< PLATFORM_OPENCL >::Pointer< t_Type >::get() const {

    return m_pointer->Get();
}


template< typename t_Type >
void Heap< PLATFORM_OPENCL >::Pointer< t_Type >::Assign( element_type const& data ) const {

    m_pointer->Set( data );
}


template< typename t_Type >
void Heap< PLATFORM_OPENCL >::Pointer< t_Type >::Assign( size_t const index, element_type const& data ) const {

    m_pointer->Set( index, data );
}


template< typename t_Type >
Heap< PLATFORM_OPENCL >::Pointer< t_Type > const& Heap< PLATFORM_OPENCL >::Pointer< t_Type >::operator=( Pointer const& other ) {

    if ( m_pointer != other.m_pointer ) {

        reset();
        m_pointer = other.m_pointer;
        if ( m_pointer != NULL )
            m_pointer->AddReference();
    }

    return *this;
}


template< typename t_Type >
Heap< PLATFORM_OPENCL >::Pointer< t_Type >::operator bool() const {

    return( m_pointer != NULL );
}


template< typename t_Type >
typename Heap< PLATFORM_OPENCL >::Object< t_Type >::Type const* Heap< PLATFORM_OPENCL >::Pointer< t_Type >::operator->() const {

    return m_pointer->Get();
}


template< typename t_Type >
typename Heap< PLATFORM_OPENCL >::Object< t_Type >::Type const& Heap< PLATFORM_OPENCL >::Pointer< t_Type >::operator*() const {

    return *m_pointer->Get();
}


template< typename t_Type >
typename Heap< PLATFORM_OPENCL >::Object< t_Type >::Type const& Heap< PLATFORM_OPENCL >::Pointer< t_Type >::operator[]( size_t const index ) const {

    return *m_pointer->Get( index );
}


template< typename t_Type >
bool Heap< PLATFORM_OPENCL >::Pointer< t_Type >::operator==( Pointer const& other ) const {

    return( m_pointer == other.m_pointer );
}


template< typename t_Type >
bool Heap< PLATFORM_OPENCL >::Pointer< t_Type >::operator!=( Pointer const& other ) const {

    return( m_pointer != other.m_pointer );
}


template< typename t_Type >
Heap< PLATFORM_OPENCL >::Pointer< t_Type >::Pointer( Object< t_Type >* pointer ) : m_pointer( pointer ) {

    if ( m_pointer != NULL )
        m_pointer->AddReference();
}


template< typename t_Type >
typename Heap< PLATFORM_OPENCL >::Pointer< t_Type >::OpenCLType Heap< PLATFORM_OPENCL >::Pointer< t_Type >::OpenCLConvert() const {

    return( ( m_pointer != NULL ) ? m_pointer->Span().first : OPENCL_NULL );
}




/*==============================================================================
    Heap< PLATFORM_OPENCL >::ObjectComparison methods
==============================================================================*/


bool Heap< PLATFORM_OPENCL >::ObjectComparison::operator()( ObjectBase const* const pLeft, ObjectBase const* const pRight ) const {

    assert( ( pLeft != NULL ) && ( pRight != NULL ) );
    return( pLeft->Span() < pRight->Span() );
}




/*==============================================================================
    Heap< PLATFORM_OPENCL > methods
==============================================================================*/


Heap< PLATFORM_OPENCL >::Heap( std::shared_ptr< OpenCL::State > const& pState, size_t const size ) : m_pState( pState ) {

    cl_int result = CL_SUCCESS;
    m_buffer.reset( clCreateBuffer( m_pState->Context(), CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR, size, NULL, &result ) );
    if ( result != CL_SUCCESS )
        THROW_OPENCL_EXCEPTION( "clCreateBuffer failed!", result );

    m_freeSpans.insert( FreeSpans::value_type( 0, size ) );
}


template< typename t_Type >
Heap< PLATFORM_OPENCL >::Pointer< t_Type > Heap< PLATFORM_OPENCL >::Allocate() {

    std::pair< size_t, size_t > const span = AllocateSpan< t_Type >();
    Object< t_Type >* pObject = new Object< t_Type >( this, span );
    MarkObject( pObject );
    return Pointer< t_Type >( pObject );
}


template< typename t_Type >
Heap< PLATFORM_OPENCL >::Pointer< t_Type[] > Heap< PLATFORM_OPENCL >::Allocate( size_t const count ) {

    std::pair< size_t, size_t > const span = AllocateSpan< t_Type >( count );
    Object< t_Type[] >* pObject = new Object< t_Type[] >( this, span, count );
    MarkObject( pObject );
    return Pointer< t_Type[] >( pObject );
}


void Heap< PLATFORM_OPENCL >::Synchronize() {

    if ( ! m_markedObjects.empty() ) {

        MarkedObjects::size_type const size = m_markedObjects.size();
        std::unique_ptr< ObjectBase const* > markedObjects( new ObjectBase const*[ size ] );
        std::copy( m_markedObjects.begin(), m_markedObjects.end(), markedObjects.get() );
        std::sort( markedObjects.get(), markedObjects.get() + size, ObjectComparison() );

        ObjectBase const* const* ii    = markedObjects.get();
        ObjectBase const* const* iiEnd = markedObjects.get() + size;
        while ( ii != iiEnd ) {

            ObjectBase const* const* iiNext = ii;
            std::pair< size_t, size_t > span = ( *ii )->Span();
            for ( ++iiNext; iiNext != iiEnd; ++iiNext ) {

                std::pair< size_t, size_t > const spanNext = ( *iiNext )->Span();
                assert( span.first + span.second <= spanNext.first );
                if ( span.first + span.second == spanNext.first )
                    span.second += spanNext.second;
                else
                    break;
            }

            cl_int result = CL_SUCCESS;
            void* const buffer = clEnqueueMapBuffer(
                m_pState->Queue(),
                m_buffer.get(),
                CL_TRUE,    // blocking
                CL_MAP_WRITE,
                span.first,
                span.second,
                0,
                NULL,
                NULL,
                &result
            );
            if ( result != CL_SUCCESS )
                THROW_OPENCL_EXCEPTION( "clEnqueueMapBuffer failed!", result );

            for ( void* destination = buffer; ii != iiNext; ++ii ) {

                ( *ii )->OpenCLWrite( destination );
                destination = reinterpret_cast< uint8_t* >( destination ) + ( *ii )->Span().second;
            }

            clEnqueueUnmapMemObject(
                m_pState->Queue(),
                m_buffer.get(),
                buffer,
                0,
                NULL,
                NULL
            );

            ii = iiNext;
        }

        m_markedObjects.clear();
    }
}


Heap< PLATFORM_OPENCL >::OpenCLType Heap< PLATFORM_OPENCL >::OpenCLConvert() const {

    return m_buffer.get();
}


template< typename t_Type >
std::pair< size_t, size_t > Heap< PLATFORM_OPENCL >::AllocateSpan( size_t const count ) {

    size_t const size = sizeof( typename OpenCL::Converter< t_Type >::Type ) * count;

    FreeSpans::right_map::iterator ii = m_freeSpans.right.lower_bound( size );
    if ( ii == m_freeSpans.right.end() )
        THROW_EXCEPTION( "out of heap space!" );

    std::pair< size_t, size_t > span( ii->second, ii->first );    // reversed because this is a right_map
    m_freeSpans.right.erase( ii );

    assert( span.second >= size );
    if ( span.second > size ) {

        m_freeSpans.insert( FreeSpans::value_type( span.first + size, span.second - size ) );
        span.second = size;
    }

    return span;
}


void Heap< PLATFORM_OPENCL >::FreeObject( ObjectBase const* pObject ) {

    m_markedObjects.erase( pObject );

    {   std::pair< size_t, size_t > span = pObject->Span();

        FreeSpans::left_map::iterator ii = m_freeSpans.left.lower_bound( span.first );

        if ( ii != m_freeSpans.left.begin() ) {

            FreeSpans::left_map::iterator iiPrev = ii;
            --iiPrev;

            assert( iiPrev->first + iiPrev->second <= span.first );
            if ( iiPrev->first + iiPrev->second == span.first ) {

                span.first  -= iiPrev->second;
                span.second += iiPrev->second;
                m_freeSpans.left.erase( iiPrev );
            }
        }

        if ( ii != m_freeSpans.left.end() ) {

            assert( span.first + span.second <= ii->first );
            if ( span.first + span.second == ii->first ) {

                span.second += ii->second;
                m_freeSpans.left.erase( ii );
            }
        }

        m_freeSpans.insert( FreeSpans::value_type( span.first, span.second ) );
    }
}


void Heap< PLATFORM_OPENCL >::MarkObject( ObjectBase const* pObject ) {

    m_markedObjects.insert( pObject );
}




}    // namespace Synchronized




#endif    // SETTING_OPENCL




#endif    // SYNCHRONIZED_HEAP_OPENCL_H