Support string starts with number

Author: TolyaTalamanov

src/tokenizer.cpp

@@ -29,9 +29,35 @@ Tokenizer::Token Tokenizer::GetToken() {
     if (c == '/') {
         _lasttok.reset(new Tokenizer::Token{Slash{}});
         // NB: Move cursor to the next symbol
-        _in->get();
+        (void)_in->get();
     } else if (std::isdigit(c) || c == '-') {
-        _lasttok.reset(new Tokenizer::Token{Double{ParseDouble()}});
+        // NB: Move cursor to the next symbol
+        (void)_in->get();
+
+        std::string accum;
+        accum += c;
+        bool met_dot    = false;
+        bool not_number = false;
+        while (true) {
+            auto ch = _in->peek();
+            if (ch == '/' || ch == '#' || std::isspace(ch) || ch == EOF) {
+                break;
+            }
+            if (!std::isdigit(ch)) {
+                if (ch == '.' && !met_dot) {
+                    met_dot = true;
+                } else {
+                    not_number = true;
+                }
+            }
+            accum += ch;
+            _in->get();
+        }
+        if (not_number) {
+            _lasttok.reset(new Tokenizer::Token{String{accum}});
+        } else {
+            _lasttok.reset(new Tokenizer::Token{Double{std::stod(accum)}});
+        }
     } else if (std::isalpha(c)) {
         auto word = ParseString();
         _lasttok.reset(new Tokenizer::Token{String{word}});

tests/test_tokenizer.cpp

@@ -12,6 +12,16 @@ TEST(TokenizerTests, EmptyStream) {
 }
 
 TEST(TokenizerTests, Double) {
+    std::stringstream ss{"42.0"};
+    Tokenizer t(&ss);
+
+    auto tok = t.GetToken();
+
+    EXPECT_TRUE(std::holds_alternative<Tokenizer::Double>(tok));
+    EXPECT_EQ(42, std::get<Tokenizer::Double>(tok).val);
+}
+
+TEST(TokenizerTests, Int) {
     std::stringstream ss{"42"};
     Tokenizer t(&ss);
 
@@ -21,6 +31,49 @@ TEST(TokenizerTests, Double) {
     EXPECT_EQ(42, std::get<Tokenizer::Double>(tok).val);
 }
 
+TEST(TokenizerTests, Negative) {
+    std::stringstream ss{"-42.5"};
+    Tokenizer t(&ss);
+
+    auto tok = t.GetToken();
+
+    EXPECT_TRUE(std::holds_alternative<Tokenizer::Double>(tok));
+    EXPECT_EQ(-42.5, std::get<Tokenizer::Double>(tok).val);
+}
+
+TEST(TokenizerTests, StringWithDot) {
+    std::stringstream ss{"42.0.0"};
+    Tokenizer t(&ss);
+
+    auto tok = t.GetToken();
+
+    EXPECT_TRUE(std::holds_alternative<Tokenizer::String>(tok));
+    EXPECT_EQ("42.0.0", std::get<Tokenizer::String>(tok).str);
+}
+
+TEST(TokenizerTests, StringStartsWithNumber) {
+    std::stringstream ss{"1212_foo"};
+    Tokenizer t(&ss);
+
+    EXPECT_EQ("1212_foo", std::get<Tokenizer::String>(t.GetToken()).str);
+}
+
+TEST(TokenizerTests, NumbersWithSlashes) {
+    std::stringstream ss{"0/1/2/"};
+    Tokenizer t(&ss);
+
+    for (int i = 0; i < 3; ++i) {
+        auto tok = t.GetToken();
+        ASSERT_EQ(i, std::get<Tokenizer::Double>(tok).val);
+        t.Next();
+        tok = t.GetToken();
+        ASSERT_TRUE(std::holds_alternative<Tokenizer::Slash>(tok));
+        t.Next();
+    }
+
+    EXPECT_TRUE(t.IsEnd());
+}
+
 TEST(TokenizerTests, String) {
     std::stringstream ss{"foo"};
     Tokenizer t(&ss);
@@ -121,10 +174,3 @@ TEST(TokenizerTests, NextLineBeforeEnd) {
 
     EXPECT_TRUE(t.IsEnd());
 }
-
-TEST(TokenizerTests, ParseStringStartsWithNumber) {
-    std::stringstream ss{"1212_foo"};
-    Tokenizer t(&ss);
-
-    EXPECT_EQ("1212_foo", std::get<Tokenizer::String>(t.GetToken()).str);
-}