/home/gian/Progetti/falcon/core/include/falcon/tokenizer.h

Go to the documentation of this file.

/*
   FALCON - The Falcon Programming Language
   FILE: tokenizer.h

   Utility to parse complex and lengty strings.
   -------------------------------------------------------------------
   Author: Giancarlo Niccolai
   Begin: Wed, 04 Feb 2009 12:02:01 +0100

   -------------------------------------------------------------------
   (C) Copyright 2009: the FALCON developers (see list in AUTHORS file)

   See LICENSE file for licensing details.
*/

#ifndef FLC_TOKENIZER_H
#define FLC_TOKENIZER_H

#include <falcon/setup.h>
#include <falcon/types.h>
#include <falcon/string.h>
#include <falcon/sequence.h>
#include <falcon/item.h>

namespace Falcon
{

class FALCON_DYN_CLASS TokenizerParams: public BaseAlloc
{
   bool m_bGroupSep;
   bool m_bBindSep;
   bool m_bTrim;
   int32 m_nMaxToken;
   bool m_bWsIsToken;
   bool m_bReturnSep;

public:
   TokenizerParams():
      m_bGroupSep( false ),
      m_bBindSep( false ),
      m_bTrim( false ),
      m_nMaxToken( -1 ),
      m_bWsIsToken( false ),
      m_bReturnSep( false )
   {}

   TokenizerParams &groupSep( bool mode = true ) { m_bGroupSep = mode; return *this; }

   TokenizerParams &wsIsToken( bool mode = true ) { m_bWsIsToken = mode; return *this; }

   TokenizerParams &bindSep( bool mode = true ) { m_bBindSep = mode; return *this; }

   TokenizerParams &returnSep( bool mode = true ) { m_bReturnSep = mode; return *this; }


   TokenizerParams &trim( bool mode = true ) { m_bTrim = mode; return *this; }

   TokenizerParams &maxToken( int32 size ) { m_nMaxToken = size; return *this; }

   bool isGroupSep() const { return m_bGroupSep; }
   bool isBindSep() const { return m_bBindSep; }
   bool isTrim() const { return m_bTrim; }
   bool isWsToken() const { return m_bWsIsToken; }
   int32 maxToken() const { return m_nMaxToken; }
   bool isReturnSep() const { return m_bReturnSep; }
};


class FALCON_DYN_CLASS Tokenizer: public Sequence
{
   String m_separators;
   TokenizerParams m_params;

   Stream *m_input;
   bool m_bOwnStream;

   String m_temp;
   uint32 m_version;

   uint32 m_nextToken;
   bool m_hasCurrent;

public:

   Tokenizer( TokenizerParams &params, const String &seps, const String &source );
   Tokenizer( TokenizerParams &params, const String &seps, Stream *inp=0, bool bOwn = false );
   Tokenizer( const Tokenizer &other );


   virtual ~Tokenizer();

   virtual const Item &front() const;

   virtual const Item &back() const;

   virtual void clear();

   virtual bool empty() const;

   virtual bool next();

   const String &getToken() const { return m_temp; }

   virtual void rewind();

   virtual FalconData* clone() const;
   virtual void gcMark( uint32 mark ) { Sequence::gcMark( mark ); }

   void parse( const String &data );

   void parse( Stream *in, bool bOwn = false );

   bool isReady() const { return m_input != 0; }

   bool hasCurrent() const { return m_hasCurrent; }

   virtual void append( const Item& itm );
   virtual void prepend( const Item& itm );

   //========================================================
   // Iterator implementation.
   //========================================================
protected:

   virtual void getIterator( Iterator& tgt, bool tail = false ) const;
   virtual void copyIterator( Iterator& tgt, const Iterator& source ) const;
   virtual void insert( Iterator &iter, const Item &data );
   virtual void erase( Iterator &iter );
   virtual bool hasNext( const Iterator &iter ) const;
   virtual bool hasPrev( const Iterator &iter ) const;
   virtual bool hasCurrent( const Iterator &iter ) const;
   virtual bool next( Iterator &iter ) const;
   virtual bool prev( Iterator &iter ) const;
   virtual Item& getCurrent( const Iterator &iter );
   virtual Item& getCurrentKey( const Iterator &iter );
   virtual bool equalIterator( const Iterator &first, const Iterator &second ) const;
};



}

#endif

/* end of tokenizer.h */

---