scriptcomp: add underscore character as valid separator for integer, …

…float, hex, octal and binary numbers

fix: catch error for underscore next to decimal in float

add test

neverwinter/nwscript/native/scriptcomplexical.cpp

@@ -189,6 +189,11 @@ int32_t CScriptCompiler::ParseCharacterPeriod(int32_t chNext)
 	}
 	else if (m_nTokenStatus == CSCRIPTCOMPILER_TOKEN_INTEGER)
 	{
+		if (chNext == '_')
+		{
+			return STRREF_CSCRIPTCOMPILER_ERROR_UNEXPECTED_CHARACTER;
+		}
+
 		m_nTokenStatus = CSCRIPTCOMPILER_TOKEN_FLOAT;
 		m_pchToken[m_nTokenCharacters] = '.';
 		++m_nTokenCharacters;
@@ -1686,6 +1691,18 @@ int32_t CScriptCompiler::ParseNextCharacter(int32_t ch, int32_t chNext, const ch
 	}
 	else if (m_nTokenStatus == CSCRIPTCOMPILER_TOKEN_INTEGER || m_nTokenStatus == CSCRIPTCOMPILER_TOKEN_FLOAT)
 	{
+		if (ch == '_')
+		{
+			if (chNext >= '0' && chNext <= '9')
+			{
+				return 0;
+			}
+			else
+			{
+				return STRREF_CSCRIPTCOMPILER_ERROR_UNEXPECTED_CHARACTER;
+			}
+		}
+
 		int32_t nCompiledCharacters = 0;
 		// Compile all suffixes that make sense.
 		if (ch == 'f')
@@ -1707,6 +1724,29 @@ int32_t CScriptCompiler::ParseNextCharacter(int32_t ch, int32_t chNext, const ch
 			return nCompiledCharacters - 1;
 		}
 	}
+	else if ((m_nTokenStatus == CSCRIPTCOMPILER_TOKEN_HEX_INTEGER ||
+		m_nTokenStatus == CSCRIPTCOMPILER_TOKEN_BINARY_INTEGER ||
+		m_nTokenStatus == CSCRIPTCOMPILER_TOKEN_OCTAL_INTEGER) && ch == '_')
+	{
+		int nReturnValue = 0;
+
+		if (m_nTokenStatus == CSCRIPTCOMPILER_TOKEN_HEX_INTEGER)
+		{
+			nReturnValue = ((chNext >= '0' && chNext <= '9') || 
+				(chNext >= 'a' && chNext <= 'f') || 
+				(chNext >= 'A' && chNext <= 'F'));
+		}
+		else if (m_nTokenStatus == CSCRIPTCOMPILER_TOKEN_BINARY_INTEGER)
+		{
+			nReturnValue = (chNext == '0' || chNext == '1');
+		}
+		else if (m_nTokenStatus == CSCRIPTCOMPILER_TOKEN_OCTAL_INTEGER)
+		{
+			nReturnValue = (chNext >= '0' && chNext <= '7');
+		}
+
+		return nReturnValue ? 0 : STRREF_CSCRIPTCOMPILER_ERROR_UNEXPECTED_CHARACTER;
+	}
 
 	// For similar reasons, we deal with alphabetic characters and then
 	// we deal with the abrupt termination of a letter.

tests/scriptcomp/corpus/constants.nss

@@ -4,6 +4,10 @@ const int INT_BINARY_LITERAL = 0b101010;
 const int INT_OCTAL_LITERAL = 0o052;
 const int INT_NEGATIVE_LITERAL = -42;
 
+const int INT_HEX_LITERAL_SEP = 0x2_A;
+const int INT_BINARY_LITERAL_SEP = 0b10_10_10;
+const int INT_OCTAL_LITERAL_SEP = 0o0_5_2;
+
 const float FLOAT_LITERAL_FULL = 42.42f;
 const float FLOAT_LITERAL_NO_SUFFIX = 42.42;
 const float FLOAT_LITERAL_ENDS_WITH_DOT = 42.;
@@ -104,6 +108,16 @@ const int HASHED_STRING_SPECIAL_CHARACTERS = h"\"\n\\\xFF\x80";
 
 void main() 
 {
+    // Test integer literals
+    Assert(INT_HEX_LITERAL == INT_LITERAL)
+    Assert(INT_BINARY_LITERAL == INT_LITERAL)
+    Assert(INT_OCTAL_LITERAL == INT_LITERAL)
+    Assert(INT_NEGATIVE_LITERAL == -INT_LITERAL)
+
+    Assert(INT_HEX_LITERAL_SEP == INT_LITERAL)
+    Assert(INT_BINARY_LITERAL_SEP == INT_LITERAL)
+    Assert(INT_OCTAL_LITERAL_SEP == INT_LITERAL)
+
     // Test float literals
     Assert(FLOAT_LITERAL_FULL == FLOAT_LITERAL_NO_SUFFIX);
     Assert(FLOAT_LITERAL_ENDS_WITH_DOT == FLOAT_LITERAL_ENDS_WITH_DOT_SUFFIX);