Arduino/app/syntax/TokenMarker.java

/*
 * TokenMarker.java - Generic token marker
 * Copyright (C) 1998, 1999 Slava Pestov
 *
 * You may use and modify this package for any purpose. Redistribution is
 * permitted, in both source and binary form, provided that this notice
 * remains intact in all source distributions of this package.
 */

package processing.app.syntax;

import javax.swing.text.Segment;
import java.util.*;

/**
 * A token marker that splits lines of text into tokens. Each token carries
 * a length field and an indentification tag that can be mapped to a color
 * for painting that token.<p>
 *
 * For performance reasons, the linked list of tokens is reused after each
 * line is tokenized. Therefore, the return value of <code>markTokens</code>
 * should only be used for immediate painting. Notably, it cannot be
 * cached.
 *
 * @author Slava Pestov
 * @version $Id$
 *
 * @see org.gjt.sp.jedit.syntax.Token
 */
public abstract class TokenMarker
{
        /**
         * A wrapper for the lower-level <code>markTokensImpl</code> method
         * that is called to split a line up into tokens.
         * @param line The line
         * @param lineIndex The line number
         */
        public Token markTokens(Segment line, int lineIndex)
        {
                if(lineIndex >= length)
                {
                        throw new IllegalArgumentException("Tokenizing invalid line: "
                                + lineIndex);
                }

                lastToken = null;

                LineInfo info = lineInfo[lineIndex];
                LineInfo prev;
                if(lineIndex == 0)
                        prev = null;
                else
                        prev = lineInfo[lineIndex - 1];

                byte oldToken = info.token;
                byte token = markTokensImpl(prev == null ?
                        Token.NULL : prev.token,line,lineIndex);

                info.token = token;

                /*
                 * This is a foul hack. It stops nextLineRequested
                 * from being cleared if the same line is marked twice.
                 *
                 * Why is this necessary? It's all JEditTextArea's fault.
                 * When something is inserted into the text, firing a
                 * document event, the insertUpdate() method shifts the
                 * caret (if necessary) by the amount inserted.
                 *
                 * All caret movement is handled by the select() method,
                 * which eventually pipes the new position to scrollTo()
                 * and calls repaint().
                 *
                 * Note that at this point in time, the new line hasn't
                 * yet been painted; the caret is moved first.
                 *
                 * scrollTo() calls offsetToX(), which tokenizes the line
                 * unless it is being called on the last line painted
                 * (in which case it uses the text area's painter cached
                 * token list). What scrollTo() does next is irrelevant.
                 *
                 * After scrollTo() has done it's job, repaint() is
                 * called, and eventually we end up in paintLine(), whose
                 * job is to paint the changed line. It, too, calls
                 * markTokens().
                 *
                 * The problem was that if the line started a multiline
                 * token, the first markTokens() (done in offsetToX())
                 * would set nextLineRequested (because the line end
                 * token had changed) but the second would clear it
                 * (because the line was the same that time) and therefore
                 * paintLine() would never know that it needed to repaint
                 * subsequent lines.
                 *
                 * This bug took me ages to track down, that's why I wrote
                 * all the relevant info down so that others wouldn't
                 * duplicate it.
                 */
                 if(!(lastLine == lineIndex && nextLineRequested))
                        nextLineRequested = (oldToken != token);

                lastLine = lineIndex;

                addToken(0,Token.END);

                return firstToken;
        }

        /**
         * An abstract method that splits a line up into tokens. It
         * should parse the line, and call <code>addToken()</code> to
         * add syntax tokens to the token list. Then, it should return
         * the initial token type for the next line.<p>
         *
         * For example if the current line contains the start of a
         * multiline comment that doesn't end on that line, this method
         * should return the comment token type so that it continues on
         * the next line.
         *
         * @param token The initial token type for this line
         * @param line The line to be tokenized
         * @param lineIndex The index of the line in the document,
         * starting at 0
         * @return The initial token type for the next line
         */
        protected abstract byte markTokensImpl(byte token, Segment line,
                int lineIndex);

        /**
         * Returns if the token marker supports tokens that span multiple
         * lines. If this is true, the object using this token marker is
         * required to pass all lines in the document to the
         * <code>markTokens()</code> method (in turn).<p>
         *
         * The default implementation returns true; it should be overridden
         * to return false on simpler token markers for increased speed.
         */
        public boolean supportsMultilineTokens()
        {
                return true;
        }

        /**
         * Informs the token marker that lines have been inserted into
         * the document. This inserts a gap in the <code>lineInfo</code>
         * array.
         * @param index The first line number
         * @param lines The number of lines
         */
        public void insertLines(int index, int lines)
        {
                if(lines <= 0)
                        return;
                length += lines;
                ensureCapacity(length);
                int len = index + lines;
                System.arraycopy(lineInfo,index,lineInfo,len,
                        lineInfo.length - len);

                for(int i = index + lines - 1; i >= index; i--)
                {
                        lineInfo[i] = new LineInfo();
                }
        }

        /**
         * Informs the token marker that line have been deleted from
         * the document. This removes the lines in question from the
         * <code>lineInfo</code> array.
         * @param index The first line number
         * @param lines The number of lines
         */
        public void deleteLines(int index, int lines)
        {
                if (lines <= 0)
                        return;
                int len = index + lines;
                length -= lines;
                System.arraycopy(lineInfo,len,lineInfo,
                        index,lineInfo.length - len);
        }

        /**
         * Returns the number of lines in this token marker.
         */
        public int getLineCount()
        {
                return length;
        }

        /**
         * Returns true if the next line should be repainted. This
         * will return true after a line has been tokenized that starts
         * a multiline token that continues onto the next line.
         */
        public boolean isNextLineRequested()
        {
                return nextLineRequested;
        }

        // protected members

        /**
         * The first token in the list. This should be used as the return
         * value from <code>markTokens()</code>.
         */
        protected Token firstToken;

        /**
         * The last token in the list. New tokens are added here.
         * This should be set to null before a new line is to be tokenized.
         */
        protected Token lastToken;

        /**
         * An array for storing information about lines. It is enlarged and
         * shrunk automatically by the <code>insertLines()</code> and
         * <code>deleteLines()</code> methods.
         */
        protected LineInfo[] lineInfo;

        /**
         * The number of lines in the model being tokenized. This can be
         * less than the length of the <code>lineInfo</code> array.
         */
        protected int length;

        /**
         * The last tokenized line.
         */
        protected int lastLine;

        /**
         * True if the next line should be painted.
         */
        protected boolean nextLineRequested;

        /**
         * Creates a new <code>TokenMarker</code>. This DOES NOT create
         * a lineInfo array; an initial call to <code>insertLines()</code>
         * does that.
         */
        protected TokenMarker()
        {
                lastLine = -1;
        }

        /**
         * Ensures that the <code>lineInfo</code> array can contain the
         * specified index. This enlarges it if necessary. No action is
         * taken if the array is large enough already.<p>
         *
         * It should be unnecessary to call this under normal
         * circumstances; <code>insertLine()</code> should take care of
         * enlarging the line info array automatically.
         *
         * @param index The array index
         */
        protected void ensureCapacity(int index)
        {
                if(lineInfo == null)
                        lineInfo = new LineInfo[index + 1];
                else if(lineInfo.length <= index)
                {
                        LineInfo[] lineInfoN = new LineInfo[(index + 1) * 2];
                        System.arraycopy(lineInfo,0,lineInfoN,0,
                                         lineInfo.length);
                        lineInfo = lineInfoN;
                }
        }

        /**
         * Adds a token to the token list.
         * @param length The length of the token
         * @param id The id of the token
         */
        protected void addToken(int length, byte id)
        {
                if(id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST)
                        throw new InternalError("Invalid id: " + id);

                if(length == 0 && id != Token.END)
                        return;

                if(firstToken == null)
                {
                        firstToken = new Token(length,id);
                        lastToken = firstToken;
                }
                else if(lastToken == null)
                {
                        lastToken = firstToken;
                        firstToken.length = length;
                        firstToken.id = id;
                }
                else if(lastToken.next == null)
                {
                        lastToken.next = new Token(length,id);
                        lastToken = lastToken.next;
                }
                else
                {
                        lastToken = lastToken.next;
                        lastToken.length = length;
                        lastToken.id = id;
                }
        }

        /**
         * Inner class for storing information about tokenized lines.
         */
        public class LineInfo
        {
                /**
                 * Creates a new LineInfo object with token = Token.NULL
                 * and obj = null.
                 */
                public LineInfo()
                {
                }

                /**
                 * Creates a new LineInfo object with the specified
                 * parameters.
                 */
                public LineInfo(byte token, Object obj)
                {
                        this.token = token;
                        this.obj = obj;
                }

                /**
                 * The id of the last token of the line.
                 */
                public byte token;

                /**
                 * This is for use by the token marker implementations
                 * themselves. It can be used to store anything that
                 * is an object and that needs to exist on a per-line
                 * basis.
                 */
                public Object obj;
        }
}