Fix number parsing to support underscores before x or b (seaofvoices#103)

Author: jeparlefrancais

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+* fix number parsing to support underscores before `x` in hexadecimal number and before `b` in binary numbers ([#103](https://github.com/seaofvoices/darklua/pull/103))
 * add bundling with path require mode ([#97](https://github.com/seaofvoices/darklua/pull/97))
 * upgrade full-moon parser to 0.18.1 ([#100](https://github.com/seaofvoices/darklua/pull/100))
 

src/nodes/expressions/number.rs

@@ -397,99 +397,89 @@ impl FromStr for NumberExpression {
     type Err = NumberParsingError;
 
     fn from_str(value: &str) -> Result<Self, Self::Err> {
-        let number = if value.starts_with("0x") || value.starts_with("0X") {
-            let is_x_uppercase = value
-                .chars()
-                .nth(1)
-                .map(char::is_uppercase)
-                .unwrap_or(false);
-
-            if let Some(index) = value.find('p') {
-                let exponent = value
-                    .get(index + 1..)
-                    .and_then(|string| string.parse().ok())
-                    .ok_or(Self::Err::InvalidHexadecimalExponent)?;
-                let number = u64::from_str_radix(value.get(2..index).unwrap(), 16)
-                    .map_err(|_| Self::Err::InvalidHexadecimalNumber)?;
-
-                HexNumber::new(number, is_x_uppercase).with_exponent(exponent, false)
-            } else if let Some(index) = value.find('P') {
-                let exponent = value
-                    .get(index + 1..)
-                    .map(filter_underscore)
-                    .and_then(|string| string.parse().ok())
-                    .ok_or(Self::Err::InvalidHexadecimalExponent)?;
-                let filtered = filter_underscore(value.get(2..index).unwrap());
-                let number = u64::from_str_radix(&filtered, 16)
-                    .map_err(|_| Self::Err::InvalidHexadecimalNumber)?;
-
-                HexNumber::new(number, is_x_uppercase).with_exponent(exponent, true)
-            } else {
-                let filtered = filter_underscore(value.get(2..).unwrap());
-                let number = u64::from_str_radix(&filtered, 16)
-                    .map_err(|_| Self::Err::InvalidHexadecimalNumber)?;
-
-                HexNumber::new(number, is_x_uppercase)
-            }
-            .into()
-        } else if value.starts_with("0b") || value.starts_with("0B") {
-            let is_b_uppercase = value
-                .chars()
-                .nth(1)
-                .map(char::is_uppercase)
-                .unwrap_or(false);
-
-            let filtered = filter_underscore(value);
-            let number = u64::from_str_radix(filtered.get(2..).unwrap(), 2)
-                .map_err(|_| Self::Err::InvalidBinaryNumber)?;
-
-            BinaryNumber::new(number, is_b_uppercase).into()
-        } else {
-            // in Luau, underscores are valid everywhere in a number except
-            // after a `.`
-            if value.contains("._") {
-                return Err(Self::Err::InvalidDecimalNumber);
+        let notation_prefix = value
+            .char_indices()
+            .filter(|(_, c)| *c != '_')
+            .take(2)
+            .nth(1);
+        let starts_with_zero = value.starts_with('0');
+
+        let number = match (starts_with_zero, notation_prefix) {
+            (true, Some((position, notation))) if matches!(notation, 'x' | 'X' | 'b' | 'B') => {
+                let is_uppercase = notation.is_uppercase();
+
+                if notation == 'x' || notation == 'X' {
+                    if let Some((exponent_is_uppercase, index)) = value
+                        .find('p')
+                        .map(|index| (false, index))
+                        .or_else(|| value.find('P').map(|index| (true, index)))
+                    {
+                        let exponent = value
+                            .get(index + 1..)
+                            .and_then(|string| string.parse().ok())
+                            .ok_or(Self::Err::InvalidHexadecimalExponent)?;
+                        let before_exponent = value.get(position + 1..index).unwrap();
+                        let number = u64::from_str_radix(before_exponent, 16)
+                            .map_err(|_| Self::Err::InvalidHexadecimalNumber)?;
+
+                        HexNumber::new(number, is_uppercase)
+                            .with_exponent(exponent, exponent_is_uppercase)
+                    } else {
+                        let filtered = filter_underscore(value.get(position + 1..).unwrap());
+                        let number = u64::from_str_radix(&filtered, 16)
+                            .map_err(|_| Self::Err::InvalidHexadecimalNumber)?;
+
+                        HexNumber::new(number, is_uppercase)
+                    }
+                    .into()
+                } else if notation == 'b' || notation == 'B' {
+                    let filtered = filter_underscore(value.get(position + 1..).unwrap());
+                    let number = u64::from_str_radix(&filtered, 2)
+                        .map_err(|_| Self::Err::InvalidBinaryNumber)?;
+
+                    BinaryNumber::new(number, is_uppercase).into()
+                } else {
+                    unreachable!()
+                }
             }
-
-            if let Some(index) = value.find('e') {
-                // in Luau, underscores are not valid before the exponent sign
-                if value.contains("_-") || value.contains("_+") {
-                    return Err(Self::Err::InvalidDecimalExponent);
+            _ => {
+                // in Luau, underscores are valid everywhere in a number except
+                // after a `.`
+                if value.contains("._") {
+                    return Err(Self::Err::InvalidDecimalNumber);
                 }
 
-                let exponent = value
-                    .get(index + 1..)
-                    .map(filter_underscore)
-                    .and_then(|string| string.parse().ok())
-                    .ok_or(Self::Err::InvalidDecimalExponent)?;
-                let number = value
-                    .get(0..index)
-                    .map(filter_underscore)
-                    .and_then(|string| string.parse().ok())
-                    .ok_or(Self::Err::InvalidDecimalNumber)?;
-
-                DecimalNumber::new(number).with_exponent(exponent, false)
-            } else if let Some(index) = value.find('E') {
-                let exponent: i64 = value
-                    .get(index + 1..)
-                    .map(filter_underscore)
-                    .and_then(|string| string.parse().ok())
-                    .ok_or(Self::Err::InvalidDecimalExponent)?;
-                let number = value
-                    .get(0..index)
-                    .map(filter_underscore)
-                    .and_then(|string| string.parse().ok())
-                    .ok_or(Self::Err::InvalidDecimalNumber)?;
-
-                DecimalNumber::new(number).with_exponent(exponent, true)
-            } else {
-                let number = filter_underscore(value)
-                    .parse::<f64>()
-                    .map_err(|_| Self::Err::InvalidDecimalNumber)?;
-
-                DecimalNumber::new(number)
+                if let Some((exponent_is_uppercase, index)) = value
+                    .find('e')
+                    .map(|index| (false, index))
+                    .or_else(|| value.find('E').map(|index| (true, index)))
+                {
+                    // in Luau, underscores are not valid before the exponent sign
+                    if value.contains("_-") || value.contains("_+") {
+                        return Err(Self::Err::InvalidDecimalExponent);
+                    }
+
+                    let exponent = value
+                        .get(index + 1..)
+                        .map(filter_underscore)
+                        .and_then(|string| string.parse().ok())
+                        .ok_or(Self::Err::InvalidDecimalExponent)?;
+                    let number = value
+                        .get(0..index)
+                        .map(filter_underscore)
+                        .and_then(|string| string.parse().ok())
+                        .ok_or(Self::Err::InvalidDecimalNumber)?;
+
+                    DecimalNumber::new(number).with_exponent(exponent, exponent_is_uppercase)
+                } else {
+                    let number = filter_underscore(value)
+                        .parse::<f64>()
+                        .map_err(|_| Self::Err::InvalidDecimalNumber)?;
+
+                    DecimalNumber::new(number)
+                }
+                .into()
             }
-            .into()
         };
 
         Ok(number)
@@ -606,28 +596,42 @@ mod test {
             parse_float_with_exponent_and_underscores("10_0.12_e_8") => DecimalNumber::new(100.12_f64).with_exponent(8, false),
             parse_trailing_dot_with_exponent("10.e8") => DecimalNumber::new(10_f64).with_exponent(8, false),
             parse_hex_number("0x12") => HexNumber::new(18, false),
+            parse_hex_number_with_underscore_before_x("0_x12") => HexNumber::new(18, false),
+            parse_hex_number_with_underscores_around_x("0_x_12") => HexNumber::new(18, false),
             parse_hex_number_with_underscore("0x12_13") => HexNumber::new(0x1213, false),
             parse_uppercase_hex_number("0X12") => HexNumber::new(18, true),
+            parse_uppercase_hex_number_with_underscore_before_x("0_X13") => HexNumber::new(19, true),
             parse_hex_number_with_lowercase("0x12a") => HexNumber::new(298, false),
             parse_hex_number_with_uppercase("0x12A") => HexNumber::new(298, false),
             parse_hex_number_with_mixed_case("0x1bF2A") => HexNumber::new(114_474, false),
             parse_hex_with_exponent("0x12p4") => HexNumber::new(18, false).with_exponent(4, false),
             parse_hex_with_exponent_uppercase("0xABP3") => HexNumber::new(171, false).with_exponent(3, true),
             parse_binary_zero("0b0") => BinaryNumber::new(0, false),
+            parse_binary_zero_with_underscore_before_b("0_b1") => BinaryNumber::new(1, false),
             parse_binary_zero_with_underscore("0b1010_1100") => BinaryNumber::new(0b1010_1100, false),
             parse_binary_zero_uppercase("0B0") => BinaryNumber::new(0, true),
+            parse_binary_zero_uppercase_with_underscore_before_b("0_B1") => BinaryNumber::new(1, true),
         );
 
         test_parse_errors!(
             parse_empty_string("") => NumberParsingError::InvalidDecimalNumber,
             missing_exponent_value("1e") => NumberParsingError::InvalidDecimalExponent,
+            missing_exponent_value_uppercase("1E") => NumberParsingError::InvalidDecimalExponent,
             invalid_underscore_position("1._1") => NumberParsingError::InvalidDecimalNumber,
             missing_negative_exponent_value("1e-") => NumberParsingError::InvalidDecimalExponent,
+            missing_negative_exponent_value_uppercase("1E-") => NumberParsingError::InvalidDecimalExponent,
             invalid_underscore_before_negative_exponent("1e_-1") => NumberParsingError::InvalidDecimalExponent,
             invalid_underscore_before_positive_exponent("1e_+1") => NumberParsingError::InvalidDecimalExponent,
+            invalid_underscore_before_negative_exponent_uppercase("1E_-1") => NumberParsingError::InvalidDecimalExponent,
+            invalid_underscore_before_positive_exponent_uppercase("1E_+1") => NumberParsingError::InvalidDecimalExponent,
             missing_hex_exponent_value("0x1p") => NumberParsingError::InvalidHexadecimalExponent,
             negative_hex_exponent_value("0x1p-3") => NumberParsingError::InvalidHexadecimalExponent,
+            missing_hex_exponent_value_uppercase("0x1P") => NumberParsingError::InvalidHexadecimalExponent,
+            invalid_hex_exponent_value("0x1p1Z") => NumberParsingError::InvalidHexadecimalExponent,
+            invalid_hex_exponent_value_uppercase("0x1P1Z") => NumberParsingError::InvalidHexadecimalExponent,
+            negative_hex_exponent_value_uppercase("0x1P-3") => NumberParsingError::InvalidHexadecimalExponent,
             invalid_digit_in_binary("0b190") => NumberParsingError::InvalidBinaryNumber,
+            invalid_digit_in_binary_uppercase("0B190") => NumberParsingError::InvalidBinaryNumber,
         );
     }
 

src/rules/snapshots/darklua_core__rules__remove_comments__test__remove_comments_in_code.snap

@@ -15,6 +15,7 @@ function foo
 end
 function foo:var 
     (param1 , p2 , ...  ) 
+    return 0x01  - 0b1 
 end
 
 for key , value   in pairs( variable ) do continue end

src/rules/snapshots/darklua_core__rules__remove_spaces__test__remove_spaces_in_code.snap

@@ -15,6 +15,7 @@ function foo-- foo
 end
 function foo:var-- var method
 (param1--[[string]],p2,...--[[rest]])-- end of parameters
+return 0x01--[[first]]-0b1-- number
 end
 
 for key--[[comment]],value in pairs(variable)do continue end

tests/bundle.rs

@@ -476,6 +476,7 @@ data:
     #[test]
     fn require_small_bundle_case() {
         let resources = memory_resources!(
+            "src/initialize.lua" => include_str!("./test_cases/small_bundle/initialize.lua"),
             "src/value.lua" => include_str!("./test_cases/small_bundle/value.lua"),
             "src/format.lua" => include_str!("./test_cases/small_bundle/format.lua"),
             "src/main.lua" => include_str!("./test_cases/small_bundle/main.lua"),

tests/snapshots/bundle__without_rules__bundle_without_rules_require_small_bundle_case.snap

@@ -2,13 +2,20 @@
 source: tests/bundle.rs
 expression: main
 ---
-local __DARKLUA_BUNDLE_MODULES={}do local function generateNumber()
+local __DARKLUA_BUNDLE_MODULES={}do local function initialize()
+end
+
+__DARKLUA_BUNDLE_MODULES.a=initialize
+end do
+local function generateNumber()
     return math.random(1, 9999)
 end
 
-__DARKLUA_BUNDLE_MODULES.a={
+__DARKLUA_BUNDLE_MODULES.b={
     zero = 0,
     one = 1,
+    hex = 0x10,
+    binary = 0b1010,
     number1 = generateNumber(),
     number2 = generateNumber(),
     number3 = generateNumber(),
@@ -18,11 +25,13 @@ local function format(value)
     return '[' .. tostring(value) .. ']'
 end
 
-__DARKLUA_BUNDLE_MODULES.b=format -- comment after returning format function
+__DARKLUA_BUNDLE_MODULES.c=format -- comment after returning format function
 end
-local value = __DARKLUA_BUNDLE_MODULES.a -- import value module
+local initialize = __DARKLUA_BUNDLE_MODULES.a -- import initialize module
+
+local value = __DARKLUA_BUNDLE_MODULES.b -- import value module
 
-local format = __DARKLUA_BUNDLE_MODULES.b --[[ import format module ]]
+local format = __DARKLUA_BUNDLE_MODULES.c --[[ import format module ]]
 
 print(format(value.number1 + value.number2))
 

tests/test_cases/small_bundle/initialize.lua

@@ -0,0 +1,4 @@
+local function initialize()
+end
+
+return initialize

tests/test_cases/small_bundle/main.lua

@@ -1,3 +1,5 @@
+local initialize = require("./initialize") -- import initialize module
+
 local value = require("./value") -- import value module
 
 local format = require("./format") --[[ import format module ]]

tests/test_cases/small_bundle/value.lua

@@ -5,6 +5,8 @@ end
 return {
     zero = 0,
     one = 1,
+    hex = 0x10,
+    binary = 0b1010,
     number1 = generateNumber(),
     number2 = generateNumber(),
     number3 = generateNumber(),

tests/test_cases/spaces_and_comments.lua

@@ -11,6 +11,7 @@ function foo -- foo
 end
 function foo:var -- var method
     (param1 --[[string]], p2 , ... --[[rest]] ) -- end of parameters
+    return 0x01 --[[first]] - 0b1 -- number
 end
 
 for key --[[comment]], value   in pairs( variable ) do continue end