loaders/csv_file.cpp

Go to the documentation of this file.

#ifndef         __CSVFILE_CPP__
#define         __CSVFILE_CPP__

#include        <loaders/csv_file.h>
#include        <utilities/testing_utilities.h>

#ifdef ENABLE_AUTOTESTING
        #include        <utilities/binary_data.h>
#endif

namespace nitro
{
        std::size_t                                             CSV_FILE_READ_BUFFER_SIZE( 1024 );

        const std::size_t                               START_FIELD( 1 );
        const std::size_t                               READ_DATA( 2 );
        const std::size_t                               END_QUOTED_FIELD( 3 );
        const std::size_t                               ADD_FIELD( 4 );
        const std::size_t                               TRY_READ_NEXT( 5 );
        const std::size_t                               QUOTE_WAS_FOUND( 6 );
        const std::size_t                               END_RECORD( 7 );

        /* virtual */ CSVReadable::~CSVReadable()
        {
                try
                {
                }
                catch( ... )
                {
                }
        }

        /* virtual */ CSVWritable::~CSVWritable()
        {
                try
                {
                }
                catch( ... )
                {
                }
        }

        void                                                    CSVFile::OpenFile( const char * FilePath )
        {
                try
                {
                        FileStream.Open( FilePath , nitro::FA_FILE_BINARY | nitro::FA_FILE_READ | nitro::FA_FILE_WRITE );
                }
                catch( nitro::exception e )
                {
                        throw( nitro::exception( std::string( "CSVFile::OpenFile( const char * FilePath )::" ) + e.what() , e.code() ) );
                }
                catch( ... )
                {
                        throw( nitro::exception( std::string( "CSVFile::OpenFile( const char * FilePath )::An error occured" ) , 1 ) );
                }
        }

        void                                                    CSVFile::OpenFile( const std::string & FilePath )
        {
                try
                {
                        OpenFile( FilePath.c_str() );
                }
                catch( nitro::exception e )
                {
                        throw( nitro::exception( std::string( "CSVFile::OpenFile( const std::string & FilePath )::" ) + e.what() , e.code() ) );
                }
                catch( ... )
                {
                        throw( nitro::exception( std::string( "CSVFile::OpenFile( const std::string & FilePath )::An error occured" ) , 1 ) );
                }
        }

        bool                                                    CSVFile::ReadRecord( CSVReadable * Readable , const std::size_t MODE /* = 0 */ )
        {
                try
                {
                        std::size_t                                     FilePosition;
                        std::size_t                                     ReadBytes;

                        // saving this value on stack to avoid 'ass pain' debug in multy-threaded applications
                        const std::size_t                       ReadBufferSize( CSV_FILE_READ_BUFFER_SIZE );

                        char *                                          Buffer( new char[ ReadBufferSize ] );

                        bool                                            QFieldStart( false );
                        bool                                            FieldStart( false );
                        std::size_t                                     FieldStartCursor( 0 );
                        std::size_t                                     State( START_FIELD );
                        std::size_t                                     FieldRawSize( 0 );

                        do
                        {
                                FilePosition = FileStream.Tell();

                                ReadBytes = FileStream.Read( Buffer , ReadBufferSize );

                                for( std::size_t i( 0 ) ; i < ReadBufferSize + 1 && i < ReadBytes + 1 ; i++ )
                                {
                                        if( ReadBytes == 0 || ( ReadBytes < ReadBufferSize && i == ReadBytes ) )
                                        {
                                                if( QFieldStart == true )
                                                {
                                                        throw( nitro::exception( "Illegal sintax (quote was not found)" , 1 ) );
                                                }
                                                if( FieldStart == true )
                                                {
                                                        State = ADD_FIELD;
                                                }
                                        }
                                        else
                                        {
                                                if( i == ReadBufferSize || i == ReadBytes )
                                                {
                                                        continue;
                                                }
                                        }

                                        if( State == START_FIELD )
                                        {
                                                FieldRawSize = 0;
                                                if( ( Buffer[ i ] >= 0x20 && Buffer[ i ] <= 0x21 ) ||
                                                        ( Buffer[ i ] >= 0x23 && Buffer[ i ] <= 0x2b ) || 
                                                        ( Buffer[ i ] >= 0x2d && Buffer[ i ] <= 0x7e ) || Buffer[ i ] == ',' )
                                                {
                                                        FieldStart = true;
                                                        FieldStartCursor = FilePosition + i;
                                                        State = READ_DATA;
                                                }

                                                if( Buffer[ i ] == '"' )
                                                {
                                                        FieldRawSize++;
                                                        QFieldStart = true;
                                                        FieldStartCursor = FilePosition + i;
                                                        State = READ_DATA;
                                                        continue;
                                                }
                                        }

                                        if( State == READ_DATA )
                                        {
                                                FieldRawSize++;
                                                if( Buffer[ i ] == '"' && QFieldStart == false )
                                                {
                                                        throw( nitro::exception( "Illegal sintax (quote was found)" , 1 ) );
                                                }
                                                if( Buffer[ i ] == '\r' && QFieldStart == false )
                                                {
                                                        FieldRawSize--;
                                                        State = ADD_FIELD;
                                                }
                                                if( Buffer[ i ] == '\n' && QFieldStart == false )
                                                {
                                                        throw( nitro::exception( "Illegal sintax (LF was found)" , 1 ) );
                                                }
                                                if( Buffer[ i ] == '"' && QFieldStart == true )
                                                {
                                                        if( i < ReadBytes - 1 )
                                                        {
                                                                State = QUOTE_WAS_FOUND;
                                                                continue;
                                                        }
                                                        else
                                                        {
                                                                State = ADD_FIELD;
                                                        }
                                                }
                                                if( Buffer[ i ] == ',' && FieldStart == true )
                                                {
                                                        FieldRawSize--;
                                                        State = ADD_FIELD;
                                                }
                                        }

                                        if( State == QUOTE_WAS_FOUND )
                                        {
                                                if( Buffer[ i ] == '"' )
                                                {
                                                        FieldRawSize++;
                                                        State = READ_DATA;
                                                        continue;
                                                }
                                                if( Buffer[ i ] == ',' || Buffer[ i ] == '\r' )
                                                {
                                                        State = ADD_FIELD;
                                                }
                                                else
                                                {
                                                        throw( nitro::exception( "Can't end quoted field (1)'" , 1 ) );
                                                }
                                        }

                                        if( State == END_QUOTED_FIELD )
                                        {
                                                if( Buffer[ i ] == ',' )
                                                {
                                                        State = ADD_FIELD;
                                                }
                                                else
                                                {
                                                        throw( nitro::exception( "Can't end quoted field (2)'" , 1 ) );
                                                }
                                        }

                                        if( State == ADD_FIELD )
                                        {
                                                if( FieldRawSize > 0 )
                                                {
                                                        nitro::BinaryData               Data;
                                                        std::size_t                             Decreaser( 0 );

                                                        if( QFieldStart == true )
                                                        {
                                                                Decreaser = 1;
                                                        }

                                                        if( FieldStartCursor < FilePosition )
                                                        {
                                                                // data is stored in several buffers, so let's read it in memory
                                                                char                            * FieldValue( new char [ FieldRawSize - 2 * Decreaser ] );

                                                                FileStream.Seek( FieldStartCursor + Decreaser , FA_FILE_BEGIN );

                                                                FileStream.Read( FieldValue , FieldRawSize - 2 * Decreaser );

                                                                Data.AppendData( FieldValue , FieldRawSize - 2 * Decreaser );

                                                                FileStream.Seek( FilePosition + ReadBufferSize , FA_FILE_BEGIN );

                                                                delete [] FieldValue;
                                                        }
                                                        else
                                                        {
                                                                // data is stored in one buffer, so it is already in memory
                                                                std::size_t                     BufferOffset( FieldStartCursor + Decreaser - FilePosition );

                                                                Data.AppendData( Buffer + BufferOffset , FieldRawSize - 2 * Decreaser );
                                                        }
                                                        
                                                        if( MODE & CSV_AUTO_ESCAPE )
                                                        {
                                                                nitro::BinaryData::ReplaceBuffer( Data , ( const char * )"\"\"" , ( const std::size_t )2 , ( const char * )"\"" , ( const std::size_t )1 );
                                                        }
                                                        
                                                        Readable->AddField( Data.GetBuffer() , Data.GetBufferLength() );
                                                }
                                                else
                                                {
                                                        Readable->AddField( "" , 0 );
                                                }

                                                QFieldStart = false;
                                                FieldStart = false;
                                                State = TRY_READ_NEXT;
                                        }

                                        if( State == TRY_READ_NEXT )
                                        {
                                                if( Buffer[ i ] == ',' )
                                                {
                                                        State = START_FIELD;
                                                }
                                                if( Buffer[ i ] == '\r' )
                                                {
                                                        State = END_RECORD;
                                                        continue;
                                                }
                                        }

                                        if( State == END_RECORD )
                                        {
                                                if( Buffer[ i ] == '\n' )
                                                {
                                                        // чтение закончено
                                                        State = START_FIELD;
                                                        FileStream.Seek( FilePosition + i + 1 , FA_FILE_BEGIN );

                                                        delete [] Buffer;

                                                        return( false );
                                                }
                                                else
                                                {
                                                        throw( nitro::exception( "Can't find LF symbol" , 1 ) );
                                                }
                                        }
                                }
                        }
                        while( ReadBytes == ReadBufferSize );

                        delete [] Buffer;

                        return( true );
                }
                catch( nitro::exception e )
                {
                        throw( nitro::exception( std::string( "CSVFile::ReadRecord( CSVReadable * Readable , const std::size_t MODE /* = 0 */ )::" ) + e.what() , e.code() ) );
                }
                catch( ... )
                {
                        throw( nitro::exception( std::string( "CSVFile::ReadRecord( CSVReadable * Readable , const std::size_t MODE /* = 0 */ )::An error occured" ) , 1 ) );
                }
        }

        void                                                    CSVFile::ReadHeader( const std::size_t MODE /* = 0 */  )
        {
                try
                {
                        ReadRecord( Header , MODE );
                }
                catch( nitro::exception e )
                {
                        throw( nitro::exception( std::string( "CSVFile::ReadHeader( const std::size_t MODE /* = 0 */  )::" ) + e.what() , e.code() ) );
                }
                catch( ... )
                {
                        throw( nitro::exception( std::string( "CSVFile::ReadHeader( const std::size_t MODE /* = 0 */  )::An error occured" ) , 1 ) );
                }
        }
        
        std::size_t                                             CSVFile::GetHeaderItemsCount( void )
        {
                try
                {
                        return( Header.size() );
                }
                catch( nitro::exception e )
                {
                        throw( nitro::exception( std::string( "CSVFile::GetHeaderItemsCount( void )::" ) + e.what() , e.code() ) );
                }
                catch( ... )
                {
                        throw( nitro::exception( std::string( "CSVFile::GetHeaderItemsCount( void )::An error occured" ) , 1 ) );
                }
        }
                
        nitro::BinaryData &                             CSVFile::GetHeaderItem( std::size_t i )
        {
                try
                {
                        return( Header[ i ] );
                }
                catch( nitro::exception e )
                {
                        throw( nitro::exception( std::string( "CSVFile::GetHeaderItem( std::size_t i )::" ) + e.what() , e.code() ) );
                }
                catch( ... )
                {
                        throw( nitro::exception( std::string( "CSVFile::GetHeaderItem( std::size_t i )::An error occured" ) , 1 ) );
                }
        }
        
        void                                                    CSVFile::AppendRecord( CSVWritable * Writable , const std::size_t MODE /* = 0 */ )
        {
                try
                {
                        std::size_t                                     FilePosition( FileStream.Tell() );

                        // \~russian сначала считаем сколько нам понадобится байт
                        // \~english count how many bytes do we need
                        std::size_t                                     BytesCount( 0 );

                        // \~russian смотрим что в конце файла
                        // \~english what do we have at the end of the file?

                        FileStream.Seek( 0 , FA_FILE_END );
                        std::size_t                                     EndOfFilePosition( FileStream.Tell() );
                        FileStream.Seek( EndOfFilePosition - 2 , FA_FILE_BEGIN );

                        char                                            EndOfFile[ 2 ];

                        FileStream.Read( EndOfFile , 2 );

                        bool                                            PrependCRLF( false );
                        if( EndOfFile[ 0 ] != '\r' || EndOfFile[ 1 ] != '\n' )
                        {
                                PrependCRLF = true;
                                BytesCount += 2;
                        }

                        const char                                      * Buffer;
                        std::size_t                                     BufferSize;
                        int                                                     Tmp;

                        // \~russian длина полей (нужно ли обрамлять поле кавычками)
                        // \~english fields length (should be the field bounded by braces)
                        for( std::size_t i( 0 ) ; i < Writable->GetFieldCount() ; i++ )
                        {
                                Writable->GetField( i , Buffer , BufferSize , MODE );

                                BytesCount += BufferSize;

                                if( MODE & CSV_AUTO_ESCAPE )
                                {
                                        Tmp = nitro::BinaryData::GetOccuranceCount( Buffer , BufferSize , "\"" , 1 );
                                        if( Tmp != 0 )
                                        {
                                                BytesCount += 2 + Tmp;
                                        }
                                }
                        }

                        // \~russian запятые
                        // \~english commas
                        BytesCount += Writable->GetFieldCount() - 1;

                        // \~russian добавляем CRLF
                        // \~english appending CRLF
                        BytesCount += 2;

                        nitro::BinaryData               Data;

                        if( PrependCRLF )
                        {
                                Data.Reserve( BytesCount + 2 );
                                Data.AppendData( "\r\n" );
                        }
                        else
                        {
                                Data.Reserve( BytesCount );
                        }

                        for( std::size_t i( 0 ) ; i < Writable->GetFieldCount() ; i++ )
                        {
                                Writable->GetField( i , Buffer , BufferSize , MODE );

                                Tmp = 0;
                                if( MODE & CSV_AUTO_ESCAPE )
                                {
                                        Tmp = nitro::BinaryData::GetOccuranceCount( Buffer , BufferSize , "\"" , 1 );
                                }

                                if( Tmp != 0 )
                                {
                                        Data.AppendData( '"' );
                                }

                                for( std::size_t j( 0 ) ; j < BufferSize ; j++ )
                                {
                                        if( Buffer[ j ] == '"' && ( MODE & CSV_AUTO_ESCAPE ) )
                                        {
                                                Data.AppendData( '"' );
                                        }
                                        Data.AppendData( Buffer[ j ] );
                                }

                                if( Tmp != 0 )
                                {
                                        Data.AppendData( '"' );
                                }

                                if( i < Writable->GetFieldCount() - 1 )
                                {
                                        Data.AppendData( ',' );
                                }
                                else
                                {
                                        Data.AppendData( "\r\n" , 2 );
                                }
                        }

                        FileStream.Write( Data.GetBuffer() , Data.GetBufferLength() );

                        FileStream.Seek( FilePosition , FA_FILE_BEGIN );
                }
                catch( nitro::exception e )
                {
                        throw( nitro::exception( std::string( "CSVFile::AppendRecord( CSVWritable * Writable , const std::size_t MODE /* = 0 */ )::" ) + e.what() , e.code() ) );
                }
                catch( ... )
                {
                        throw( nitro::exception( std::string( "CSVFile::AppendRecord( CSVWritable * Writable , const std::size_t MODE /* = 0 */ )::An error occured" ) , 1 ) );
                }
        }

        void                                                    CSVFile::CloseFile( void )
        {
                try
                {
                        FileStream.Close();
                }
                catch( nitro::exception e )
                {
                        throw( nitro::exception( std::string( "CSVFile::CloseFile( void )::" ) + e.what() , e.code() ) );
                }
                catch( ... )
                {
                        throw( nitro::exception( std::string( "CSVFile::CloseFile( void )::An error occured" ) , 1 ) );
                }
        }

        /* virtual */ CSVFile::~CSVFile()
        {
                try
                {
                        CloseFile();
                }
                catch( ... )
                {
                }
        }

        /*static */void                                 CSVFile::SetReadBufferSize( std::size_t ReadBufferSize )
        {
                CSV_FILE_READ_BUFFER_SIZE = ReadBufferSize;
        }

        // \~russian секция автотестирования
        // \~english unit tests

        BEGIN_TESTING_SECTION()

                ENABLE_CLASS_TESTING( CSVFile )

                CLASS_MEMBER_FUNCTION_TESTING_1( CSVFile , aliasOpenFile , tstOpenFile , const char * , void , NO_RET )
                CLASS_MEMBER_FUNCTION_TESTING_0( CSVFile , CloseFile , tstCloseFile , void , NO_RET )

                #ifdef ENABLE_AUTOTESTING
                        std::vector< std::vector< nitro::BinaryData > >         CSVRecords;
                        std::vector< nitro::BinaryData >                                        Fields;
                        nitro::BinaryData                                                                       Field;
        NITRO_EXPORTING bool                    tstReadRecord( const char * ObjectName , std::size_t MODE )
                        {
                                class   Reader : public CSVReadable{
                                public:
                                        void                                            AddField( const char * Buffer , const std::size_t & BufferSize )
                                        {
                                                Fields.push_back( nitro::BinaryData() );

                                                char                                    * NewBuffer( new char[ BufferSize + 1 ] );

                                                memset( NewBuffer , 0 , BufferSize + 1 );
                                                memcpy( NewBuffer , Buffer , BufferSize );

                                                Fields.back().AppendData( NewBuffer , BufferSize + 1 );

                                                delete [] NewBuffer;
                                        }
                                };

                                Reader          r;

                                return( CSVFileGetter( ObjectName ).ReadRecord( ( CSVReadable * ) & r , MODE ) );
                        }

                        NITRO_EXPORTING void                    tstAppendRecord( const char * ObjectName , std::size_t MODE )
                        {
                                try
                                {
                                        class   Writer : public CSVWritable{
                                        public:
                                                std::size_t                                     GetFieldCount( void )
                                                {
                                                        return( Fields.size() );
                                                }
                                                
                                                void                                            GetField( const std::size_t & i , const char * & Buffer , std::size_t & BufferSize , const std::size_t & )
                                                {
                                                        Buffer = Fields[ i ].GetBuffer();

                                                        BufferSize = Fields[ i ].GetBufferLength();
                                                }
                                        };

                                        Writer                  w;

                                        CSVFileGetter( ObjectName ).AppendRecord( ( CSVWritable * ) & w , MODE );
                                }
                                catch( nitro::exception e )
                                {
                                        std::cout<<e.what()<<std::endl;
                                }
                                catch( ... )
                                {
                                        std::cout<<"An error occured"<<std::endl;
                                }
                        }

                        NITRO_EXPORTING void                    tstAppendRecordTpl( const char * ObjectName , std::size_t MODE )
                        {
                                try
                                {
                                        CSVFileGetter( ObjectName ).AppendRecord( Fields , MODE );
                                }
                                catch( nitro::exception e )
                                {
                                        std::cout<<e.what()<<std::endl;
                                }
                                catch( ... )
                                {
                                        std::cout<<"An error occured"<<std::endl;
                                }
                        }

                        NITRO_EXPORTING void                    tstPushFields( void )
                        {
                                CSVRecords.push_back( Fields );
                        }

                        NITRO_EXPORTING void                    tstAppendAllRecords( const char * ObjectName , std::size_t MODE )
                        {
                                try
                                {
                                        CSVFileGetter( ObjectName ).AppendAllRecords( CSVRecords , MODE );
                                }
                                catch( nitro::exception e )
                                {
                                        std::cout<<e.what()<<std::endl;
                                }
                                catch( ... )
                                {
                                        std::cout<<"An error occured"<<std::endl;
                                }
                        }

                        NITRO_EXPORTING void                    tstReadAllRecords( const char * ObjectName , std::size_t MODE )
                        {
                                try
                                {
                                        std::vector< std::vector< nitro::BinaryData > >         CSVRecords;

                                        CSVFileGetter( ObjectName ).ReadAllRecords( CSVRecords , MODE );

                                        Fields = CSVRecords.back();
                                }
                                catch( nitro::exception e )
                                {
                                        std::cout<<e.what()<<std::endl;
                                }
                                catch( ... )
                                {
                                        std::cout<<"An error occured"<<std::endl;
                                }
                        }

                        NITRO_EXPORTING int                     tstGetCount( void )
                        {
                                return( ( int )( Fields.size() ) );
                        }

                        NITRO_EXPORTING const char *    tstGetField( int i )
                        {
                                Field = Fields[ i ];
                                Field.AppendData( "\0" , 1 );

                                return( Field.GetBuffer() );
                        }

                        NITRO_EXPORTING void                    tstSetField( std::size_t i , const char * Data , std::size_t DataSize )
                        {
                                if( i >= Fields.size() )
                                {
                                        Fields.resize( i + 1 );
                                }

                                Fields[ i ].Release();
                                Fields[ i ].AppendData( Data , DataSize );
                        }

                        NITRO_EXPORTING void                    tstReadRecordTpl( const char * ObjectName , std::size_t MODE )
                        {
                                try
                                {
                                        CSVFileGetter( ObjectName ).ReadRecord< std::vector< nitro::BinaryData > >( Fields , MODE );
                                }
                                catch( nitro::exception e )
                                {
                                        std::cout<<e.what()<<std::endl;
                                }
                                catch( ... )
                                {
                                        std::cout<<"An error occured"<<std::endl;
                                }
                        }
                #endif

                // \~russian тестируем статические методы класса
                // \~english testing static methods
                FUNCTION_TESTING_1( CSVFile::SetReadBufferSize , tstSetReadBufferSize , std::size_t , void , NO_RET )

        END_TESTING_SECTION()
}

#endif