Skip to content

Commit 14be0ae

Browse files
gnembongnembon
committed
extract commonly used class from the Tokenizer
1 parent 9086db9 commit 14be0ae

14 files changed

+172
-184
lines changed

‎src/main/java/carpet/script/CarpetScriptHost.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -754,7 +754,7 @@ public Value callLegacy(CommandSourceStack source, String call, List<Integer> co
754754
}
755755
}
756756
String sign = "";
757-
for (Tokenizer.Token tok : Tokenizer.simplepass(arg))
757+
for (Token tok : Tokenizer.simplepass(arg))
758758
{
759759
switch (tok.type)
760760
{

‎src/main/java/carpet/script/Expression.java

+70-70
Large diffs are not rendered by default.

‎src/main/java/carpet/script/Fluff.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ public interface ILazyFunction extends EvalNode
7070

7171
boolean numParamsVaries();
7272

73-
LazyValue lazyEval(Context c, Context.Type type, Expression expr, Tokenizer.Token token, List<LazyValue> lazyParams);
73+
LazyValue lazyEval(Context c, Context.Type type, Expression expr, Token token, List<LazyValue> lazyParams);
7474

7575
static void checkInterrupts()
7676
{
@@ -93,7 +93,7 @@ public interface ILazyOperator extends EvalNode
9393

9494
boolean isLeftAssoc();
9595

96-
LazyValue lazyEval(Context c, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2);
96+
LazyValue lazyEval(Context c, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2);
9797
}
9898

9999
public interface IOperator extends ILazyOperator
@@ -187,7 +187,7 @@ public boolean transitive()
187187
}
188188

189189
@Override
190-
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Tokenizer.Token t, List<LazyValue> lazyParams)
190+
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Token t, List<LazyValue> lazyParams)
191191
{
192192

193193
return new LazyValue()
@@ -273,7 +273,7 @@ public boolean transitive()
273273
}
274274

275275
@Override
276-
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2)
276+
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2)
277277
{
278278
return (c, typeIgnored) -> {
279279
try
@@ -308,7 +308,7 @@ public boolean transitive()
308308
}
309309

310310
@Override
311-
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Tokenizer.Token t, LazyValue v1, LazyValue v2)
311+
public LazyValue lazyEval(Context cc, Context.Type type, Expression e, Token t, LazyValue v1, LazyValue v2)
312312
{
313313
if (v2 != null)
314314
{

‎src/main/java/carpet/script/ScriptCommand.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,7 @@
3737
import net.minecraft.commands.arguments.coordinates.BlockPosArgument;
3838
import net.minecraft.core.BlockPos;
3939
import net.minecraft.server.level.ServerLevel;
40-
import net.minecraft.world.Clearable;
4140
import net.minecraft.world.level.block.Block;
42-
import net.minecraft.world.level.block.entity.BlockEntity;
4341
import net.minecraft.world.level.block.state.pattern.BlockInWorld;
4442
import net.minecraft.world.level.levelgen.structure.BoundingBox;
4543

@@ -438,7 +436,7 @@ private static int listGlobals(CommandContext<CommandSourceStack> context, boole
438436
return;
439437
}
440438
Expression expr = fun.getExpression();
441-
Tokenizer.Token tok = fun.getToken();
439+
Token tok = fun.getToken();
442440
List<String> snippet = expr.getExpressionSnippet(tok);
443441
Carpet.Messenger_message(source, "wb " + fun.fullName(), "t defined at: line " + (tok.lineno + 1) + " pos " + (tok.linepos + 1));
444442
for (String snippetLine : snippet)

‎src/main/java/carpet/script/ScriptHost.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ public boolean isDefaultApp()
138138
@FunctionalInterface
139139
public interface ErrorSnooper
140140
{
141-
List<String> apply(Expression expression, Tokenizer.Token token, Context context, String message);
141+
List<String> apply(Expression expression, Token token, Context context, String message);
142142
}
143143

144144
public ErrorSnooper errorSnooper = null;

‎src/main/java/carpet/script/Token.java

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package carpet.script;
2+
3+
public class Token {
4+
enum TokenType {
5+
FUNCTION(true, false), OPERATOR(true, false), UNARY_OPERATOR(true, false),
6+
VARIABLE(false, false), CONSTANT(false, true),
7+
LITERAL(false, true), HEX_LITERAL(false, true), STRINGPARAM(false, true),
8+
OPEN_PAREN(false, true), COMMA(false, true), CLOSE_PAREN(false, true), MARKER(false, true);
9+
10+
final boolean functional;
11+
final boolean constant;
12+
13+
TokenType(boolean functional, boolean constant) {
14+
this.functional = functional;
15+
this.constant = constant;
16+
}
17+
18+
public boolean isFunctional() {
19+
return functional;
20+
}
21+
22+
public boolean isConstant() {
23+
return constant;
24+
}
25+
}
26+
27+
public String surface = "";
28+
public TokenType type;
29+
public int pos;
30+
public int linepos;
31+
public int lineno;
32+
public static final Token NONE = new Token();
33+
34+
public Token morphedInto(TokenType newType, String newSurface) {
35+
Token created = new Token();
36+
created.surface = newSurface;
37+
created.type = newType;
38+
created.pos = pos;
39+
created.linepos = linepos;
40+
created.lineno = lineno;
41+
return created;
42+
}
43+
44+
public void morph(TokenType type, String s) {
45+
this.type = type;
46+
this.surface = s;
47+
}
48+
49+
public void append(char c) {
50+
surface += c;
51+
}
52+
53+
public void append(String s) {
54+
surface += s;
55+
}
56+
57+
public char charAt(int pos) {
58+
return surface.charAt(pos);
59+
}
60+
61+
public int length() {
62+
return surface.length();
63+
}
64+
65+
@Override
66+
public String toString() {
67+
return surface;
68+
}
69+
}

‎src/main/java/carpet/script/Tokenizer.java

+1-80
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* Expression tokenizer that allows to iterate over a {@link String}
1515
* expression token by token. Blank characters will be skipped.
1616
*/
17-
public class Tokenizer implements Iterator<Tokenizer.Token>
17+
public class Tokenizer implements Iterator<Token>
1818
{
1919
/**
2020
* What character to use for decimal separators.
@@ -421,83 +421,4 @@ public void remove()
421421
throw new InternalExpressionException("remove() not supported");
422422
}
423423

424-
public static class Token
425-
{
426-
enum TokenType
427-
{
428-
FUNCTION(true, false), OPERATOR(true, false), UNARY_OPERATOR(true, false),
429-
VARIABLE(false, false), CONSTANT(false, true),
430-
LITERAL(false, true), HEX_LITERAL(false, true), STRINGPARAM(false, true),
431-
OPEN_PAREN(false, true), COMMA(false, true), CLOSE_PAREN(false, true), MARKER(false, true);
432-
433-
final boolean functional;
434-
final boolean constant;
435-
436-
TokenType(boolean functional, boolean constant)
437-
{
438-
this.functional = functional;
439-
this.constant = constant;
440-
}
441-
442-
public boolean isFunctional()
443-
{
444-
return functional;
445-
}
446-
447-
public boolean isConstant()
448-
{
449-
return constant;
450-
}
451-
}
452-
453-
public String surface = "";
454-
public TokenType type;
455-
public int pos;
456-
public int linepos;
457-
public int lineno;
458-
public static final Token NONE = new Token();
459-
460-
public Token morphedInto(TokenType newType, String newSurface)
461-
{
462-
Token created = new Token();
463-
created.surface = newSurface;
464-
created.type = newType;
465-
created.pos = pos;
466-
created.linepos = linepos;
467-
created.lineno = lineno;
468-
return created;
469-
}
470-
471-
public void morph(TokenType type, String s)
472-
{
473-
this.type = type;
474-
this.surface = s;
475-
}
476-
477-
public void append(char c)
478-
{
479-
surface += c;
480-
}
481-
482-
public void append(String s)
483-
{
484-
surface += s;
485-
}
486-
487-
public char charAt(int pos)
488-
{
489-
return surface.charAt(pos);
490-
}
491-
492-
public int length()
493-
{
494-
return surface.length();
495-
}
496-
497-
@Override
498-
public String toString()
499-
{
500-
return surface;
501-
}
502-
}
503424
}

‎src/main/java/carpet/script/exception/ExpressionException.java

+8-8
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import carpet.script.Context;
44
import carpet.script.Expression;
5-
import carpet.script.Tokenizer;
5+
import carpet.script.Token;
66
import carpet.script.external.Carpet;
77
import carpet.script.value.FunctionValue;
88

@@ -16,7 +16,7 @@
1616
public class ExpressionException extends StacklessRuntimeException implements ResolvedException
1717
{
1818
public final Context context;
19-
public final Tokenizer.Token token;
19+
public final Token token;
2020
public final List<FunctionValue> stack = new ArrayList<>();
2121
private final Supplier<String> lazyMessage;
2222
private String cachedMessage = null;
@@ -28,15 +28,15 @@ public static void prepareForDoom()
2828

2929
public ExpressionException(Context c, Expression e, String message)
3030
{
31-
this(c, e, Tokenizer.Token.NONE, message);
31+
this(c, e, Token.NONE, message);
3232
}
3333

34-
public ExpressionException(Context c, Expression e, Tokenizer.Token t, String message)
34+
public ExpressionException(Context c, Expression e, Token t, String message)
3535
{
3636
this(c, e, t, message, Collections.emptyList());
3737
}
3838

39-
public ExpressionException(Context c, Expression e, Tokenizer.Token t, String message, List<FunctionValue> stack)
39+
public ExpressionException(Context c, Expression e, Token t, String message, List<FunctionValue> stack)
4040
{
4141
super("Error");
4242
this.stack.addAll(stack);
@@ -45,7 +45,7 @@ public ExpressionException(Context c, Expression e, Tokenizer.Token t, String me
4545
context = c;
4646
}
4747

48-
public ExpressionException(Context c, Expression e, Tokenizer.Token t, Supplier<String> messageSupplier, List<FunctionValue> stack)
48+
public ExpressionException(Context c, Expression e, Token t, Supplier<String> messageSupplier, List<FunctionValue> stack)
4949
{
5050
super("Error");
5151
this.stack.addAll(stack);
@@ -54,7 +54,7 @@ public ExpressionException(Context c, Expression e, Tokenizer.Token t, Supplier<
5454
context = c;
5555
}
5656

57-
private static List<String> makeError(Expression expr, @Nullable Tokenizer.Token token, String errmessage)
57+
private static List<String> makeError(Expression expr, @Nullable Token token, String errmessage)
5858
{
5959
List<String> errMsg = new ArrayList<>();
6060
errmessage += expr.getModuleName() == null ? "" : (" in " + expr.getModuleName());
@@ -76,7 +76,7 @@ private static List<String> makeError(Expression expr, @Nullable Tokenizer.Token
7676
return errMsg;
7777
}
7878

79-
static synchronized String makeMessage(Context c, Expression e, Tokenizer.Token t, String message) throws ExpressionException
79+
static synchronized String makeMessage(Context c, Expression e, Token t, String message) throws ExpressionException
8080
{
8181
if (c.getErrorSnooper() != null)
8282
{

‎src/main/java/carpet/script/exception/InternalExpressionException.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import carpet.script.Context;
44
import carpet.script.Expression;
5-
import carpet.script.Tokenizer;
5+
import carpet.script.Token;
66
import carpet.script.value.FunctionValue;
77

88
import java.util.ArrayList;
@@ -30,7 +30,7 @@ public InternalExpressionException(String message)
3030
* @return The new {@link ExpressionException} (or {@link ProcessedThrowStatement}),
3131
* depending on the implementation.
3232
*/
33-
public ExpressionException promote(Context c, Expression e, Tokenizer.Token token)
33+
public ExpressionException promote(Context c, Expression e, Token token)
3434
{
3535
return new ExpressionException(c, e, token, getMessage(), stack);
3636
}

‎src/main/java/carpet/script/exception/ProcessedThrowStatement.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
import carpet.script.Context;
66
import carpet.script.Expression;
7-
import carpet.script.Tokenizer.Token;
7+
import carpet.script.Token;
88
import carpet.script.value.FunctionValue;
99
import carpet.script.value.Value;
1010

‎src/main/java/carpet/script/exception/ThrowStatement.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import carpet.script.Context;
44
import carpet.script.Expression;
5-
import carpet.script.Tokenizer.Token;
5+
import carpet.script.Token;
66
import carpet.script.value.StringValue;
77
import carpet.script.value.Value;
88

‎src/main/java/carpet/script/language/Functions.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
import carpet.script.Expression;
55
import carpet.script.Fluff;
66
import carpet.script.LazyValue;
7-
import carpet.script.Tokenizer;
7+
import carpet.script.Token;
88
import carpet.script.argument.FunctionArgument;
99
import carpet.script.exception.InternalExpressionException;
1010
import carpet.script.exception.ReturnStatement;
@@ -45,7 +45,7 @@ public static void apply(Expression expression) // public just to get the javado
4545
expression.addCustomFunction("call", new Fluff.AbstractLazyFunction(-1, "call")
4646
{
4747
@Override
48-
public LazyValue lazyEval(Context c, Context.Type t, Expression expr, Tokenizer.Token tok, List<LazyValue> lv)
48+
public LazyValue lazyEval(Context c, Context.Type t, Expression expr, Token tok, List<LazyValue> lv)
4949
{
5050
if (lv.isEmpty())
5151
{

0 commit comments

Comments
 (0)