LQCD Monitoring Database

%w{token}.each{|library| require library}

module Yapser

  class Tokenizer

    # Current Token index, zero-based.

    attr_accessor :position

    # Highest Token index reached since last time that maxPosition was set.

    attr_accessor :maxPosition

    # Symbols recognized by the tokenizer are:

    #   Comma, Dot, Integer, In, Is, Leftbrace, Leftbracket, Leftparen,

    #   Name, Number, Ordinal, Nil, True, False,

    #   Rightbrace, Rightbracket, Rightparen, String and Version.

    # Tokens are in the form of Token objects

    # EOD is indicated by a nil response.

    # The token returned is the first Token of the maximum string length

    # recognized in the @@patterns variable.

    # The @@patterns variable contains an Array of two element Arrays.

    # In the two element arrays, the first element is the token type

    # and the second element is the recognizing regular expression.

    # A Token kind value of false means that that text is to be ignored.

    # The :error Token kind at the end is to find bad characters.

    # N.B., the numeric portion of this taken with \Z added at the right

    # end of the pattern is present in the parser for the prettyPrint

    # routine's use (for a requirement to print numbers unquoted).

    @@patterns = [

        [false,          %r{\A\s\s*}],

        [false,          %r{\A#.*$}],

        [false,          %r{\A//.*$}],

        [:true,          %r{\Atrue((?=[^A-Za-z0-9_])|(?=\Z))}],

        [:false,         %r{\Afalse((?=[^A-Za-z0-9_])|(?=\Z))}],

        [:nil,           %r{\Anil((?=[^A-Za-z0-9_])|(?=\Z))}],

        [:nil,           %r{\Anull((?=[^A-Za-z0-9_])|(?=\Z))}],

        [:include,       %r{\A\$remember((?=[^A-Za-z0-9_])|(?=\Z))}],

        [:exclude,       %r{\A\$forget((?=[^A-Za-z0-9_])|(?=\Z))}],

        [:number,        %r{\A[-+]?infinity((?=[^A-Za-z0-9_])|(?=\Z))}],

        [:number,        %r{\A[-+]?[0-9]+\.[0-9]*[Ee][-+]?[0-9]{1,3}

                            ((?=[^0-9])|(?=\Z))}x],

        [:number,        %r{\A[-+]?\.[0-9]+[Ee][-+]?[0-9]{1,3}

                            ((?=[^0-9])|(?=\Z))}x],

        [:number,        %r{\A[-+]?[0-9]+[Ee][-+]?[0-9]{1,3}

                            ((?=[^0-9])|(?=\Z))}x],

        [:number,        %r{\A[-+]?[0-9]+\.[0-9]*}],

        [:number,        %r{\A[-+]?\.[0-9]+}],

        [:integer,       %r{\A[-+][0-9][0-9]*}],

        [:ordinal,       %r{\A[0-9][0-9]*}],

        [:name,          %r{\A[A-Za-z_][A-Za-z0-9_]*}],

        [:string,        %r{\A"([^"\\]|\\["\\])*"}],

        [:dot,           %r{\A\.}],

        [:in,            %r{\A::}],

        [:leftbrace,     %r{\A\{}],

        [:rightbrace,    %r{\A\}}],

        [:leftbracket,   %r{\A\[}],

        [:rightbracket,  %r{\A\]}],

        [:leftparen,     %r{\A\(}],

        [:rightparen,    %r{\A\)}],

        [:comma,         %r{\A\,}],

        [:is,            %r{\A\:}],

        [:error,         %r{\A.}]

      ]

    def initialize(text,patterns=nil)

      @line = 1

      @ordinal = 0

      @position = 0

      @maxPosition = 0

      @text = text

      @patterns = patterns||@@patterns

      @tokens = []

      if text.class.to_s == "String"

        while true do

          break if @text.empty?

          matched = false

          theKind = nil

          theText = ""

          for kind, regEx in @patterns do

            if @text =~ regEx

              if matched

                if theText.length < $&.length

                  theKind = kind

                  theText = $&

                end

              else

                matched = true

                theKind = kind

                theText = $&

              end

            end

          end

          if matched

            @text = @text[theText.length..-1]

            if theKind

              @ordinal += 1

              @tokens.push(Token.new(theKind, theText, @line, @ordinal))

            end

            # Ruby 1.8 does not have String.chars

            for c in theText.split("")

              if c == "\n"

                @line += 1

                @ordinal = 0

              end

            end

          else

            @tokens = [Token.new(:error, @text, @line, @ordinal + 1)]

            @text = ""

          end

        end

      else

        @tokens = [Token.new(:error,text,@line,@ordinal)]

      end

    end

    def setMaxPosition

      @maxPosition = @position

    end

    def backup(amount=1)

      @position -= amount

    end

    def get

      copy(look {

        setMaxPosition if @maxPosition < @position

        @position += 1

      })

    end

    def peek

      copy(look { })

    end

    def [](location) avail(location) end

    def numberOfTokens

      @tokens.size

    end

    def copyOfTokens

      copy(@tokens)

    end

    def abstract()

      result = []

      @tokens.each{|token| result << [token.kind, token.text.dup]}

      result

    end

    private

    def copy(thing) Marshal.load(Marshal.dump(thing)) end

    def avail(location)

      token = @tokens[location] rescue nil if location >= 0

      copy(token)

    end

    def look(&block)

      (@position < @tokens.length) \

        ? ( result = @tokens[@position]; block.call; result ) : nil

    end

  end

end