Commit 07b00196 authored by Custom2043's avatar Custom2043
Browse files

Fix and better tokens data handling

parent 6759db84
No preview for this file type
......@@ -29,10 +29,6 @@ public class Compiler
"try", "void", "volatile", "while"
};
public static final String variableNames[] = {
"column", "length", "line", "start", "total", "totalTokens", "builderSearches"
};
public static final String forbiddenTokenNames[] = {
"ArraySearch", "BracketSearch", "Builder", "OptionalSearch", "OrSearch", "ParserString", "Result", "Search"
};
......@@ -82,10 +78,6 @@ public class Compiler
if (s.equals(((Line)t).header.id.getName()))
report(new Info(InfoModel.illegalIdentifier, ((Line)t).header.id));
for (String s : variableNames)
if (s.equals(((Line)t).header.id.getName()))
report(new Info(InfoModel.illegalIdentifier, ((Line)t).header.id));
for (String s : forbiddenTokenNames)
if (s.equals(((Line)t).header.id.getName()))
report(new Info(InfoModel.illegalIdentifier, ((Line)t).header.id));
......
......@@ -17,7 +17,7 @@ public class Info
@Override
public String toString()
{
return String.format("%s : %s at line %d, column %d, char %d : %s", this.model.etat.name(), this.model.message, this.token.line, this.token.column, this.token.start, this.token.total.replace("\n", "").replace("\r", ""));
return String.format("%s : %s at line %d, column %d, char %d : %s", this.model.etat.name(), this.model.message, this.token.getLine(), this.token.getColumn(), this.token.getStart(), this.token.getPrinteableTotal());
}
}
enum InfoModel
......
......@@ -131,9 +131,9 @@ public class ParserStringGenerator
else
{
parserCharacterBuilder.append("0");
parserCharacterBuilder.append(((NumberChar)character).base.total);
parserCharacterBuilder.append(((NumberChar)character).base.getTotal());
for (ParserString p : ((NumberChar)character).value)
parserCharacterBuilder.append(p.total);
parserCharacterBuilder.append(p.getTotal());
}
}
else
......
......@@ -55,17 +55,6 @@ public class Parser
}
}
}
/**
* @return the line of the character (used to report errors)
*/
private static int getLine(int atChar)
{
int currentLine = 1;
for (Integer i : lineJumps)
if (i < atChar)
currentLine ++;
return currentLine;
}
/**
* Reads until the line defining the given token (inclusive) or until the end if the builder is null</br>
......@@ -86,12 +75,12 @@ public class Parser
int oldIndex = index;
Token t;
ParserMemory memory = ParserMemory.setLastResult(getTokenStart(), line);
ParserMemory memory = ParserMemory.setLastResult(index, line);
mark.currentToken = memory.getToken();
if (memory.getToken() != null)
{
//System.out.println("Found : "+mark.token+" at : "+index+" while building "+builder);
index = mark.currentToken.start + mark.currentToken.length;
index = mark.currentToken.getStart() + mark.currentToken.getLength();
mark();
}
......@@ -102,7 +91,7 @@ public class Parser
if ((t = buildToken(lineBuilder, mark.currentToken)) != null)
{
mark.currentToken = t;
ParserMemory.addResult(getTokenStart(), new ParserMemory(mark.currentToken, lineBuilder.getLine()));
ParserMemory.addResult(mark.currentToken.getStart(), new ParserMemory(mark.currentToken, lineBuilder.getLine()));
}
else if (lineBuilder.getLine() == line && !excluded)
{
......@@ -126,6 +115,8 @@ public class Parser
//System.out.println("Try to build : "+tokenBuilder+" with "+current+" at index : "+index);
mark();
int start = current == null ? index : current.getStart();
Result[] results = new Result[tokenBuilder.getSearches().length];
......@@ -144,7 +135,7 @@ public class Parser
for (int i=0;i<results.length;i++)
TokenCreator.fill(total, results[i], build, new LinkedList<Integer>(), new LinkedList<Integer>());
build.totalTokens = total.toArray(new Token[total.size()]);
build.setDatas(start, index-start, getLine(start), Parser.getTokenStartColumn(start), Parser.readFrom(start, index-start), total.toArray(new Token[total.size()]));
build.getClass().getDeclaredField("searchResults").set(build, results);
......@@ -167,6 +158,8 @@ public class Parser
*/
static ParserString readParserString(int stringIndex)
{
int start = index;
Parser.submark();
int indexCheck = -1;
......@@ -186,14 +179,14 @@ public class Parser
if (correct)
{
ParserString string = new ParserString(stringIndex);
string.totalTokens = new Token[0];
string.setDatas(start, index - start, getLine(start), Parser.getTokenStartColumn(start), Parser.readFrom(start, index-start), new Token[0]);
return string;
}
Parser.subreset();
return null;
}
public static char read() throws ArrayIndexOutOfBoundsException
private static char read() throws ArrayIndexOutOfBoundsException
{
return content[index++];
}
......@@ -207,50 +200,53 @@ public class Parser
index = mark.markIndex;
mark.submarks = new LinkedList<>();
}
public static boolean isSubmarked()
static boolean isSubmarked()
{
return !mark.submarks.isEmpty();
}
public static void submark()
static void submark()
{
mark.submarks.addLast(index);
}
public static void subreset()
static void subreset()
{
index = mark.submarks.removeLast();
}
public static int getTokenStart()
{
return mark.currentToken == null ? mark.markIndex : mark.currentToken.start;
}
public static int getTokenStartLine()
/**
* @return the line of the character (used to report errors)
*/
public static int getLine(int atChar)
{
return getLine(getTokenStart());
int currentLine = 1;
for (Integer i : lineJumps)
if (i < atChar)
currentLine ++;
return currentLine;
}
public static int getTokenStartColumn()
public static int getTokenStartColumn(int tokenStart)
{
int currentIndex = -1;
for (Integer i : lineJumps)
if (i <= getTokenStart())
if (i <= tokenStart)
currentIndex = i;
return getTokenStart() - currentIndex;
return tokenStart - currentIndex;
}
public static boolean moreToRead()
private static boolean moreToRead()
{
return index < content.length;
}
public static int getTokenLength()
{
return index - getTokenStart();
}
public static String readFrom(int start, int length)
private static String readFrom(int start, int length)
{
char[] c = new char[length];
for (int i=start ; i<start+length;i++)
c[i - start] = content[i];
return new String(c);
}
private static class Mark
{
private int markIndex = index;
......
......@@ -88,6 +88,6 @@ public class ParserString extends Token
@Override
public String toString()
{
return "ParserString "+this.index+", start "+this.start+" ; length : "+this.length+" : "+this.getPrinteableTotal();
return "ParserString "+this.index+", start "+this.getStart()+" ; length : "+this.getLength()+" : "+this.getPrinteableTotal();
}
}
......@@ -46,26 +46,52 @@ public abstract class Token
/********** Do not modify or remove **********/
public final int start, length, line, column;
public final String total;
Token[] totalTokens;
public Token()
private int start, length, line, column;
private String total;
private Token[] totalTokens;
void setDatas(int s, int l, int li, int c, String t, Token[] tokens)
{
this.start = Parser.getTokenStart();
this.length = Parser.getTokenLength();
this.line = Parser.getTokenStartLine();
this.column = Parser.getTokenStartColumn();
this.total = Parser.readFrom(this.start, this.length);
this.start = s;
this.length = l;
this.line = li;
this.column = c;
this.total = t;
this.totalTokens = tokens;
}
@Override
public String toString()
public int getStart()
{
return this.toString(0);
return this.start;
}
public int getLength()
{
return this.length;
}
public int getLine()
{
return this.line;
}
public int getColumn()
{
return this.column;
}
public String getTotal()
{
return this.total;
}
public Token[] getTotalTokens()
{
return this.totalTokens;
}
@Override
public String toString()
{
return this.toString(0);
}
public String toString(int i)
{
String s = "";
......@@ -80,6 +106,10 @@ public abstract class Token
s += "\n" + t.toString(i+1);
return s;
}
public String getPrinteableTotal()
{
return this.total.replace("\n", "\\n").replace("\t", "\\t").replace("\r", "\\r");
}
public static String getStringFromTokens(Token... tokens)
{
......@@ -88,12 +118,7 @@ public abstract class Token
builder.append(t.total);
return builder.toString();
}
public String getPrinteableTotal()
{
return this.total.replace("\n", "\\n").replace("\t", "\\t").replace("\r", "\\r");
}
public List<Token> getTokensIn(Class<? extends Token> c)
{
List<Token> tokens = new LinkedList<>();
......
......@@ -52,7 +52,7 @@ public class CharacterToken extends Token
public ParserCharacter toParserCharacter()
{
if (this.character1.length == 1 && this.character1[0].total.equals("*") && this.character2[0] == null)
if (this.character1.length == 1 && this.character1[0].getTotal().equals("*") && this.character2[0] == null)
return this.notToken == null ? ParserCharacter.any : ParserCharacter.none;
char[] start = new char[this.character1.length], end = new char[this.character1.length];
......
......@@ -39,10 +39,10 @@ public class Identifier extends Token
public String getName()
{
StringBuilder builder = new StringBuilder();
builder.append(this.firstLetter.total);
builder.append(this.firstLetter.getTotal());
if (this.idName != null)
for(Token t : this.idName)
builder.append(t.total);
builder.append(t.getTotal());
return builder.toString();
}
}
......@@ -40,11 +40,11 @@ public class NumberChar extends Token
public char toChar()
{
if (this.base.index == 4)
return (char)Integer.parseInt(this.total.substring(2), 2);
return (char)Integer.parseInt(this.getTotal().substring(2), 2);
else if (this.base.index == 6)
return (char)Integer.parseInt(this.total.substring(2), 16);
return (char)Integer.parseInt(this.getTotal().substring(2), 16);
else if (this.base.index == 8)
return (char)Integer.parseInt(this.total.substring(2), 8);
return (char)(int)Integer.valueOf(this.total);
return (char)Integer.parseInt(this.getTotal().substring(2), 8);
return (char)(int)Integer.valueOf(this.getTotal());
}
}
......@@ -43,8 +43,8 @@ public class SingleChar extends Token
public String getString()
{
if (this.backslash != null)
return "\\"+this.backslash.total;
return this.total;
return "\\"+this.backslash.getTotal();
return this.getTotal();
}
public char toChar()
......@@ -67,7 +67,7 @@ public class SingleChar extends Token
return '\"';
}
else
return ((ParserString)this.searchResults[0].getToken()).total.charAt(0);
return ((ParserString)this.searchResults[0].getToken()).getTotal().charAt(0);
return 0;//Shouldn't happen
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment