Commit 07b00196 authored by Custom2043's avatar Custom2043
Browse files

Fix and better tokens data handling

parent 6759db84
No preview for this file type
...@@ -29,10 +29,6 @@ public class Compiler ...@@ -29,10 +29,6 @@ public class Compiler
"try", "void", "volatile", "while" "try", "void", "volatile", "while"
}; };
public static final String variableNames[] = {
"column", "length", "line", "start", "total", "totalTokens", "builderSearches"
};
public static final String forbiddenTokenNames[] = { public static final String forbiddenTokenNames[] = {
"ArraySearch", "BracketSearch", "Builder", "OptionalSearch", "OrSearch", "ParserString", "Result", "Search" "ArraySearch", "BracketSearch", "Builder", "OptionalSearch", "OrSearch", "ParserString", "Result", "Search"
}; };
...@@ -82,10 +78,6 @@ public class Compiler ...@@ -82,10 +78,6 @@ public class Compiler
if (s.equals(((Line)t).header.id.getName())) if (s.equals(((Line)t).header.id.getName()))
report(new Info(InfoModel.illegalIdentifier, ((Line)t).header.id)); report(new Info(InfoModel.illegalIdentifier, ((Line)t).header.id));
for (String s : variableNames)
if (s.equals(((Line)t).header.id.getName()))
report(new Info(InfoModel.illegalIdentifier, ((Line)t).header.id));
for (String s : forbiddenTokenNames) for (String s : forbiddenTokenNames)
if (s.equals(((Line)t).header.id.getName())) if (s.equals(((Line)t).header.id.getName()))
report(new Info(InfoModel.illegalIdentifier, ((Line)t).header.id)); report(new Info(InfoModel.illegalIdentifier, ((Line)t).header.id));
......
...@@ -17,7 +17,7 @@ public class Info ...@@ -17,7 +17,7 @@ public class Info
@Override @Override
public String toString() public String toString()
{ {
return String.format("%s : %s at line %d, column %d, char %d : %s", this.model.etat.name(), this.model.message, this.token.line, this.token.column, this.token.start, this.token.total.replace("\n", "").replace("\r", "")); return String.format("%s : %s at line %d, column %d, char %d : %s", this.model.etat.name(), this.model.message, this.token.getLine(), this.token.getColumn(), this.token.getStart(), this.token.getPrinteableTotal());
} }
} }
enum InfoModel enum InfoModel
......
...@@ -131,9 +131,9 @@ public class ParserStringGenerator ...@@ -131,9 +131,9 @@ public class ParserStringGenerator
else else
{ {
parserCharacterBuilder.append("0"); parserCharacterBuilder.append("0");
parserCharacterBuilder.append(((NumberChar)character).base.total); parserCharacterBuilder.append(((NumberChar)character).base.getTotal());
for (ParserString p : ((NumberChar)character).value) for (ParserString p : ((NumberChar)character).value)
parserCharacterBuilder.append(p.total); parserCharacterBuilder.append(p.getTotal());
} }
} }
else else
......
...@@ -55,17 +55,6 @@ public class Parser ...@@ -55,17 +55,6 @@ public class Parser
} }
} }
} }
/**
* @return the line of the character (used to report errors)
*/
private static int getLine(int atChar)
{
int currentLine = 1;
for (Integer i : lineJumps)
if (i < atChar)
currentLine ++;
return currentLine;
}
/** /**
* Reads until the line defining the given token (inclusive) or until the end if the builder is null</br> * Reads until the line defining the given token (inclusive) or until the end if the builder is null</br>
...@@ -86,12 +75,12 @@ public class Parser ...@@ -86,12 +75,12 @@ public class Parser
int oldIndex = index; int oldIndex = index;
Token t; Token t;
ParserMemory memory = ParserMemory.setLastResult(getTokenStart(), line); ParserMemory memory = ParserMemory.setLastResult(index, line);
mark.currentToken = memory.getToken(); mark.currentToken = memory.getToken();
if (memory.getToken() != null) if (memory.getToken() != null)
{ {
//System.out.println("Found : "+mark.token+" at : "+index+" while building "+builder); //System.out.println("Found : "+mark.token+" at : "+index+" while building "+builder);
index = mark.currentToken.start + mark.currentToken.length; index = mark.currentToken.getStart() + mark.currentToken.getLength();
mark(); mark();
} }
...@@ -102,7 +91,7 @@ public class Parser ...@@ -102,7 +91,7 @@ public class Parser
if ((t = buildToken(lineBuilder, mark.currentToken)) != null) if ((t = buildToken(lineBuilder, mark.currentToken)) != null)
{ {
mark.currentToken = t; mark.currentToken = t;
ParserMemory.addResult(getTokenStart(), new ParserMemory(mark.currentToken, lineBuilder.getLine())); ParserMemory.addResult(mark.currentToken.getStart(), new ParserMemory(mark.currentToken, lineBuilder.getLine()));
} }
else if (lineBuilder.getLine() == line && !excluded) else if (lineBuilder.getLine() == line && !excluded)
{ {
...@@ -126,6 +115,8 @@ public class Parser ...@@ -126,6 +115,8 @@ public class Parser
//System.out.println("Try to build : "+tokenBuilder+" with "+current+" at index : "+index); //System.out.println("Try to build : "+tokenBuilder+" with "+current+" at index : "+index);
mark(); mark();
int start = current == null ? index : current.getStart();
Result[] results = new Result[tokenBuilder.getSearches().length]; Result[] results = new Result[tokenBuilder.getSearches().length];
...@@ -144,7 +135,7 @@ public class Parser ...@@ -144,7 +135,7 @@ public class Parser
for (int i=0;i<results.length;i++) for (int i=0;i<results.length;i++)
TokenCreator.fill(total, results[i], build, new LinkedList<Integer>(), new LinkedList<Integer>()); TokenCreator.fill(total, results[i], build, new LinkedList<Integer>(), new LinkedList<Integer>());
build.totalTokens = total.toArray(new Token[total.size()]); build.setDatas(start, index-start, getLine(start), Parser.getTokenStartColumn(start), Parser.readFrom(start, index-start), total.toArray(new Token[total.size()]));
build.getClass().getDeclaredField("searchResults").set(build, results); build.getClass().getDeclaredField("searchResults").set(build, results);
...@@ -167,6 +158,8 @@ public class Parser ...@@ -167,6 +158,8 @@ public class Parser
*/ */
static ParserString readParserString(int stringIndex) static ParserString readParserString(int stringIndex)
{ {
int start = index;
Parser.submark(); Parser.submark();
int indexCheck = -1; int indexCheck = -1;
...@@ -186,14 +179,14 @@ public class Parser ...@@ -186,14 +179,14 @@ public class Parser
if (correct) if (correct)
{ {
ParserString string = new ParserString(stringIndex); ParserString string = new ParserString(stringIndex);
string.totalTokens = new Token[0]; string.setDatas(start, index - start, getLine(start), Parser.getTokenStartColumn(start), Parser.readFrom(start, index-start), new Token[0]);
return string; return string;
} }
Parser.subreset(); Parser.subreset();
return null; return null;
} }
public static char read() throws ArrayIndexOutOfBoundsException private static char read() throws ArrayIndexOutOfBoundsException
{ {
return content[index++]; return content[index++];
} }
...@@ -207,50 +200,53 @@ public class Parser ...@@ -207,50 +200,53 @@ public class Parser
index = mark.markIndex; index = mark.markIndex;
mark.submarks = new LinkedList<>(); mark.submarks = new LinkedList<>();
} }
public static boolean isSubmarked() static boolean isSubmarked()
{ {
return !mark.submarks.isEmpty(); return !mark.submarks.isEmpty();
} }
public static void submark() static void submark()
{ {
mark.submarks.addLast(index); mark.submarks.addLast(index);
} }
public static void subreset() static void subreset()
{ {
index = mark.submarks.removeLast(); index = mark.submarks.removeLast();
} }
public static int getTokenStart()
{ /**
return mark.currentToken == null ? mark.markIndex : mark.currentToken.start; * @return the line of the character (used to report errors)
} */
public static int getTokenStartLine() public static int getLine(int atChar)
{ {
return getLine(getTokenStart()); int currentLine = 1;
for (Integer i : lineJumps)
if (i < atChar)
currentLine ++;
return currentLine;
} }
public static int getTokenStartColumn() public static int getTokenStartColumn(int tokenStart)
{ {
int currentIndex = -1; int currentIndex = -1;
for (Integer i : lineJumps) for (Integer i : lineJumps)
if (i <= getTokenStart()) if (i <= tokenStart)
currentIndex = i; currentIndex = i;
return getTokenStart() - currentIndex; return tokenStart - currentIndex;
} }
public static boolean moreToRead()
private static boolean moreToRead()
{ {
return index < content.length; return index < content.length;
} }
public static int getTokenLength()
{ private static String readFrom(int start, int length)
return index - getTokenStart();
}
public static String readFrom(int start, int length)
{ {
char[] c = new char[length]; char[] c = new char[length];
for (int i=start ; i<start+length;i++) for (int i=start ; i<start+length;i++)
c[i - start] = content[i]; c[i - start] = content[i];
return new String(c); return new String(c);
} }
private static class Mark private static class Mark
{ {
private int markIndex = index; private int markIndex = index;
......
...@@ -88,6 +88,6 @@ public class ParserString extends Token ...@@ -88,6 +88,6 @@ public class ParserString extends Token
@Override @Override
public String toString() public String toString()
{ {
return "ParserString "+this.index+", start "+this.start+" ; length : "+this.length+" : "+this.getPrinteableTotal(); return "ParserString "+this.index+", start "+this.getStart()+" ; length : "+this.getLength()+" : "+this.getPrinteableTotal();
} }
} }
...@@ -46,26 +46,52 @@ public abstract class Token ...@@ -46,26 +46,52 @@ public abstract class Token
/********** Do not modify or remove **********/ /********** Do not modify or remove **********/
public final int start, length, line, column; private int start, length, line, column;
public final String total; private String total;
Token[] totalTokens; private Token[] totalTokens;
public Token()
void setDatas(int s, int l, int li, int c, String t, Token[] tokens)
{ {
this.start = Parser.getTokenStart(); this.start = s;
this.length = Parser.getTokenLength(); this.length = l;
this.line = Parser.getTokenStartLine(); this.line = li;
this.column = Parser.getTokenStartColumn(); this.column = c;
this.total = Parser.readFrom(this.start, this.length); this.total = t;
this.totalTokens = tokens;
} }
@Override
public String toString() public int getStart()
{ {
return this.toString(0); return this.start;
}
public int getLength()
{
return this.length;
}
public int getLine()
{
return this.line;
} }
public int getColumn()
{
return this.column;
}
public String getTotal()
{
return this.total;
}
public Token[] getTotalTokens() public Token[] getTotalTokens()
{ {
return this.totalTokens; return this.totalTokens;
} }
@Override
public String toString()
{
return this.toString(0);
}
public String toString(int i) public String toString(int i)
{ {
String s = ""; String s = "";
...@@ -80,6 +106,10 @@ public abstract class Token ...@@ -80,6 +106,10 @@ public abstract class Token
s += "\n" + t.toString(i+1); s += "\n" + t.toString(i+1);
return s; return s;
} }
public String getPrinteableTotal()
{
return this.total.replace("\n", "\\n").replace("\t", "\\t").replace("\r", "\\r");
}
public static String getStringFromTokens(Token... tokens) public static String getStringFromTokens(Token... tokens)
{ {
...@@ -88,12 +118,7 @@ public abstract class Token ...@@ -88,12 +118,7 @@ public abstract class Token
builder.append(t.total); builder.append(t.total);
return builder.toString(); return builder.toString();
} }
public String getPrinteableTotal()
{
return this.total.replace("\n", "\\n").replace("\t", "\\t").replace("\r", "\\r");
}
public List<Token> getTokensIn(Class<? extends Token> c) public List<Token> getTokensIn(Class<? extends Token> c)
{ {
List<Token> tokens = new LinkedList<>(); List<Token> tokens = new LinkedList<>();
......
...@@ -52,7 +52,7 @@ public class CharacterToken extends Token ...@@ -52,7 +52,7 @@ public class CharacterToken extends Token
public ParserCharacter toParserCharacter() public ParserCharacter toParserCharacter()
{ {
if (this.character1.length == 1 && this.character1[0].total.equals("*") && this.character2[0] == null) if (this.character1.length == 1 && this.character1[0].getTotal().equals("*") && this.character2[0] == null)
return this.notToken == null ? ParserCharacter.any : ParserCharacter.none; return this.notToken == null ? ParserCharacter.any : ParserCharacter.none;
char[] start = new char[this.character1.length], end = new char[this.character1.length]; char[] start = new char[this.character1.length], end = new char[this.character1.length];
......
...@@ -39,10 +39,10 @@ public class Identifier extends Token ...@@ -39,10 +39,10 @@ public class Identifier extends Token
public String getName() public String getName()
{ {
StringBuilder builder = new StringBuilder(); StringBuilder builder = new StringBuilder();
builder.append(this.firstLetter.total); builder.append(this.firstLetter.getTotal());
if (this.idName != null) if (this.idName != null)
for(Token t : this.idName) for(Token t : this.idName)
builder.append(t.total); builder.append(t.getTotal());
return builder.toString(); return builder.toString();
} }
} }
...@@ -40,11 +40,11 @@ public class NumberChar extends Token ...@@ -40,11 +40,11 @@ public class NumberChar extends Token
public char toChar() public char toChar()
{ {
if (this.base.index == 4) if (this.base.index == 4)
return (char)Integer.parseInt(this.total.substring(2), 2); return (char)Integer.parseInt(this.getTotal().substring(2), 2);
else if (this.base.index == 6) else if (this.base.index == 6)
return (char)Integer.parseInt(this.total.substring(2), 16); return (char)Integer.parseInt(this.getTotal().substring(2), 16);
else if (this.base.index == 8) else if (this.base.index == 8)
return (char)Integer.parseInt(this.total.substring(2), 8); return (char)Integer.parseInt(this.getTotal().substring(2), 8);
return (char)(int)Integer.valueOf(this.total); return (char)(int)Integer.valueOf(this.getTotal());
} }
} }
...@@ -43,8 +43,8 @@ public class SingleChar extends Token ...@@ -43,8 +43,8 @@ public class SingleChar extends Token
public String getString() public String getString()
{ {
if (this.backslash != null) if (this.backslash != null)
return "\\"+this.backslash.total; return "\\"+this.backslash.getTotal();
return this.total; return this.getTotal();
} }
public char toChar() public char toChar()
...@@ -67,7 +67,7 @@ public class SingleChar extends Token ...@@ -67,7 +67,7 @@ public class SingleChar extends Token
return '\"'; return '\"';
} }
else else
return ((ParserString)this.searchResults[0].getToken()).total.charAt(0); return ((ParserString)this.searchResults[0].getToken()).getTotal().charAt(0);
return 0;//Shouldn't happen return 0;//Shouldn't happen
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment