Char literals (#512)

Add lexing, parsing, typechecking, and compilation support for character literals. Right now they're fairly useless without actually being related to Strings at all, but that'll come in a future change once this groundwork is laid.
kengorab · Nov 28, 2024 · 1601beb · 1601beb
1 parent 2b893b8
commit 1601beb
Show file tree

Hide file tree

Showing 22 changed files with 334 additions and 5 deletions.
diff --git a/projects/compiler/example.abra b/projects/compiler/example.abra
@@ -1 +1,11 @@
-println(1)
+val x = 'a'
+println(x)
+
+val t = x == 'a'
+val f = x == 'A'
+println(t, f)
+
+val h = 'a'.hash()
+println(h)
+
+println(x.asInt())
diff --git a/projects/compiler/src/compiler.abra b/projects/compiler/src/compiler.abra
@@ -1873,6 +1873,7 @@ export type Compiler {
       LiteralAstNode.Int(v) => (Value.Int(v), Type(kind: TypeKind.PrimitiveInt))
       LiteralAstNode.Float(f) => (Value.Float(f), Type(kind: TypeKind.PrimitiveFloat))
       LiteralAstNode.Bool(b) => (Value.Int(if b 1 else 0), Type(kind: TypeKind.PrimitiveBool))
+      LiteralAstNode.Char(c) => (Value.IntU64(c), Type(kind: TypeKind.PrimitiveChar))
       LiteralAstNode.String(s) => {
         val dataPtr = self._builder.buildGlobalString(s.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"").replaceAll("\n", "\\n").replaceAll("\r", "\\r"))
         val instancePtr = try self._constructString(dataPtr, Value.Int(s.length))
@@ -2266,7 +2267,7 @@ export type Compiler {
   }
 
   func _compileEqLogic(self, leftVal: Value, rightVal: Value, ty: Type, position: Position, localName: String? = None, negate = false): Result<Value, CompileError> {
-    if self._typeIsInt(ty) || self._typeIsFloat(ty) || self._typeIsBool(ty) {
+    if self._typeIsInt(ty) || self._typeIsFloat(ty) || self._typeIsBool(ty) || self._typeIsChar(ty) {
       val res = if negate {
         val res = try self._currentFn.block.buildCompareNeq(leftVal, rightVal, localName) else |e| return qbeError(e)
         self._currentFn.block.buildExt(res, false)
@@ -2416,6 +2417,7 @@ export type Compiler {
       TypeKind.PrimitiveInt => Ok(Some(QbeType.U64))
       TypeKind.PrimitiveFloat => Ok(Some(QbeType.F64))
       TypeKind.PrimitiveBool => Ok(Some(QbeType.U64))
+      TypeKind.PrimitiveChar => Ok(Some(QbeType.U64))
       TypeKind.PrimitiveString => Ok(Some(QbeType.Pointer))
       TypeKind.Never => Ok(None)
       TypeKind.Any => todo("TypeKind.Any")
@@ -3079,6 +3081,17 @@ export type Compiler {
 
         Ok(str)
       }
+      "char_as_int" => {
+        self._currentFn.block.addComment("begin char_as_int...")
+
+        val _arg = if arguments[0] |arg| arg else unreachable("'char_as_int' has 1 required argument")
+        val arg = if _arg |arg| arg else unreachable("'char_as_int' has 1 required argument")
+        val argVal = try self._compileExpression(arg)
+
+        self._currentFn.block.addComment("...char_as_int end")
+
+        Ok(argVal)
+      }
       "int_as_float" => {
         self._currentFn.block.addComment("begin int_as_float...")
 
@@ -3389,6 +3402,7 @@ export type Compiler {
         if struct == self._project.preludeFloatStruct return self._getOrCompileFloatToStringMethod()
         if struct == self._project.preludeStringStruct return self._getOrCompileStringToStringMethod()
         if struct == self._project.preludeBoolStruct return self._getOrCompileBoolToStringMethod()
+        if struct == self._project.preludeCharStruct return self._getOrCompileCharToStringMethod()
 
         val _fn = struct.instanceMethods.find(m => m.label.name == "toString")
         val fn = if _fn |f| f else unreachable("every struct has a toString method defined")
@@ -3699,6 +3713,35 @@ export type Compiler {
     Ok(fnVal)
   }
 
+  func _getOrCompileCharToStringMethod(self): Result<QbeFunction, CompileError> {
+    val charTypeQbe = try self._getQbeTypeForTypeExpect(Type(kind: TypeKind.PrimitiveChar), "char qbe type should exist")
+    val stringTypeQbe = try self._getQbeTypeForTypeExpect(Type(kind: TypeKind.PrimitiveString), "string qbe type should exist")
+
+    val typeName = try self._structTypeName(self._project.preludeCharStruct)
+    val methodName = "$typeName..toString"
+    if self._builder.getFunction(methodName) |fn| return Ok(fn)
+
+    val fnVal = self._builder.buildFunction(name: methodName, returnType: Some(stringTypeQbe))
+    val prevFn = self._currentFn
+    self._currentFn = fnVal
+
+    fnVal.addComment("Char#toString(self): String")
+    val selfParam = fnVal.addParameter("self", stringTypeQbe)
+
+    val lowestByteVal = try self._currentFn.block.buildAnd(selfParam, Value.Int(0xff)) else |e| return qbeError(e)
+    val strData = try self._callMalloc(Value.Int(1))
+    self._currentFn.block.buildStoreL(lowestByteVal, strData)
+
+    val strVal = try self._constructString(strData, Value.Int(1))
+    fnVal.block.buildReturn(Some(strVal))
+
+    try fnVal.block.verify() else |e| return qbeError(e)
+
+    self._currentFn = prevFn
+
+    Ok(fnVal)
+  }
+
   func _getOrCompileStringToStringMethod(self): Result<QbeFunction, CompileError> {
     val stringTypeQbe = try self._getQbeTypeForTypeExpect(Type(kind: TypeKind.PrimitiveString), "string qbe type should exist")
 
@@ -3792,6 +3835,28 @@ export type Compiler {
 
           return Ok(fnVal)
         }
+        if struct == self._project.preludeCharStruct {
+          val charTypeQbe = try self._getQbeTypeForTypeExpect(Type(kind: TypeKind.PrimitiveChar), "char qbe type should exist")
+
+          val methodName = "Char..eq"
+          if self._builder.getFunction(methodName) |fn| return Ok(fn)
+
+          val fnVal = self._builder.buildFunction(name: methodName, returnType: Some(boolTypeQbe))
+          val prevFn = self._currentFn
+          self._currentFn = fnVal
+
+          fnVal.addComment("Char#eq(self, other: Char): Char")
+          val selfParam = fnVal.addParameter("self", charTypeQbe)
+          val otherParam = fnVal.addParameter("other", charTypeQbe)
+          val res = try self._currentFn.block.buildCompareEq(selfParam, otherParam) else |e| return qbeError(e)
+          fnVal.block.buildReturn(Some(self._currentFn.block.buildExt(res, false)))
+
+          try fnVal.block.verify() else |e| return qbeError(e)
+
+          self._currentFn = prevFn
+
+          return Ok(fnVal)
+        }
 
         val _fn = struct.instanceMethods.find(m => m.label.name == "eq")
         val fn = if _fn |f| f else unreachable("every struct has an eq method defined")
@@ -4007,6 +4072,26 @@ export type Compiler {
 
           return Ok(fnVal)
         }
+        if struct == self._project.preludeCharStruct {
+          val charTypeQbe = try self._getQbeTypeForTypeExpect(Type(kind: TypeKind.PrimitiveChar), "char qbe type should exist")
+
+          val methodName = "Char..hash"
+          if self._builder.getFunction(methodName) |fn| return Ok(fn)
+
+          val fnVal = self._builder.buildFunction(name: methodName, returnType: Some(intTypeQbe))
+          val prevFn = self._currentFn
+          self._currentFn = fnVal
+
+          fnVal.addComment("Charhash(self): Int")
+          val selfParam = fnVal.addParameter("self", charTypeQbe)
+          fnVal.block.buildReturn(Some(selfParam))
+
+          try fnVal.block.verify() else |e| return qbeError(e)
+
+          self._currentFn = prevFn
+
+          return Ok(fnVal)
+        }
 
         val _fn = struct.instanceMethods.find(m => m.label.name == "hash")
         val fn = if _fn |f| f else unreachable("every struct has a hash method defined")
@@ -4296,6 +4381,7 @@ export type Compiler {
       TypeKind.PrimitiveInt => Ok((StructOrEnum.Struct(self._project.preludeIntStruct), []))
       TypeKind.PrimitiveFloat => Ok((StructOrEnum.Struct(self._project.preludeFloatStruct), []))
       TypeKind.PrimitiveBool => Ok((StructOrEnum.Struct(self._project.preludeBoolStruct), []))
+      TypeKind.PrimitiveChar => Ok((StructOrEnum.Struct(self._project.preludeCharStruct), []))
       TypeKind.PrimitiveString => Ok((StructOrEnum.Struct(self._project.preludeStringStruct), []))
       TypeKind.Never => unreachable("getInstanceTypeForType: Never")
       TypeKind.Generic(name) => {
@@ -4629,6 +4715,7 @@ export type Compiler {
   func _typeIsInt(self, ty: Type): Bool = ty.kind == TypeKind.PrimitiveInt || ty.kind == TypeKind.Instance(StructOrEnum.Struct(self._project.preludeIntStruct), [])
   func _typeIsFloat(self, ty: Type): Bool = ty.kind == TypeKind.PrimitiveFloat || ty.kind == TypeKind.Instance(StructOrEnum.Struct(self._project.preludeFloatStruct), [])
   func _typeIsBool(self, ty: Type): Bool = ty.kind == TypeKind.PrimitiveBool || ty.kind == TypeKind.Instance(StructOrEnum.Struct(self._project.preludeBoolStruct), [])
+  func _typeIsChar(self, ty: Type): Bool = ty.kind == TypeKind.PrimitiveChar || ty.kind == TypeKind.Instance(StructOrEnum.Struct(self._project.preludeCharStruct), [])
   func _typeIsString(self, ty: Type): Bool = ty.kind == TypeKind.PrimitiveString || ty.kind == TypeKind.Instance(StructOrEnum.Struct(self._project.preludeStringStruct), [])
 
   // TODO: this is copied from Typechecker

diff --git a/projects/compiler/src/lexer.abra b/projects/compiler/src/lexer.abra
@@ -12,6 +12,7 @@ export enum TokenKind {
   Int(value: Int)
   Float(value: Float)
   Bool(value: Bool)
+  Char(ch: Int)
   String(value: String)
   StringInterpolation(chunks: StringInterpolationChunk[])
 
@@ -89,6 +90,7 @@ export enum TokenKind {
     TokenKind.Bool(value) => value.toString()
     TokenKind.String => "string"
     TokenKind.StringInterpolation => "string"
+    TokenKind.Char => "char"
     TokenKind.Ident(name) => if name == "_" { "_" } else "identifier"
     TokenKind.If => "if"
     TokenKind.Else => "else"
@@ -251,6 +253,8 @@ export type Lexer {
       try self._tokenizeInteger(startPos: position)
     } else if ch == "\"" {
       try self._tokenizeString()
+    } else if ch == "'" {
+      try self._tokenizeChar()
     } else if ch.isAlpha() || ch == "_" {
       self._tokenizeIdentifier(startPos: position)
     } else if ch == "/" && (peek == "/" || peek == "*") {
@@ -435,6 +439,27 @@ export type Lexer {
     Ok(Token(position: startPos, kind: TokenKind.Float(float)))
   }
 
+  func _tokenizeChar(self): Result<Token, LexerError> {
+    var startPos = self._curPos()
+    self._advance() // consume "'"
+
+    var ch = self._input[self._cursor]
+    if ch == "'" return Err(LexerError(position: self._curPos(), kind: LexerErrorKind.UnexpectedChar(ch)))
+    if ch == "\\" todo("character escape sequences")
+
+    // TODO: once characters are supported, there should be no need for low-level byte manipulation like this
+    val intVal = ch._buffer.offset(0).load().asInt()
+    self._advance()
+
+    ch = self._input[self._cursor]
+    if ch != "'" {
+      return Err(LexerError(position: self._curPos(), kind: LexerErrorKind.UnexpectedChar(ch)))
+    }
+    self._advance() // consume "'"
+
+    Ok(Token(position: startPos, kind: TokenKind.Char(intVal)))
+  }
+
   func _tokenizeString(self): Result<Token, LexerError> {
     var startPos = self._curPos()
     val origStartPos = startPos

diff --git a/projects/compiler/src/parser.abra b/projects/compiler/src/parser.abra
@@ -25,6 +25,7 @@ export enum LiteralAstNode {
   Int(value: Int)
   Float(value: Float)
   Bool(value: Bool)
+  Char(value: Int)
   String(value: String)
 }
 
@@ -1092,6 +1093,7 @@ export type Parser {
       TokenKind.Int => self._parseLiteral()
       TokenKind.Float => self._parseLiteral()
       TokenKind.Bool => self._parseLiteral()
+      TokenKind.Char => self._parseLiteral()
       TokenKind.String => self._parseLiteral()
       TokenKind.StringInterpolation => self._parseStringInterpolation()
       TokenKind.LParen => self._parseGroupedOrTupleOrLambda()
@@ -1129,6 +1131,7 @@ export type Parser {
       TokenKind.Int(value) => LiteralAstNode.Int(value)
       TokenKind.Float(value) => LiteralAstNode.Float(value)
       TokenKind.Bool(value) => LiteralAstNode.Bool(value)
+      TokenKind.Char(value) => LiteralAstNode.Char(value)
       TokenKind.String(value) => LiteralAstNode.String(value)
       TokenKind.StringInterpolation => {
         return Err(ParseError(position: token.position, kind: ParseErrorKind.NotYetImplemented))
@@ -1313,6 +1316,7 @@ export type Parser {
     val items = try self._commaSeparated(end: TokenKind.RBrace, consumeFinal: true, fn: () => {
       val keyToken = try self._expectPeek()
       val key = match keyToken.kind {
+        TokenKind.Char => try self._parseExpression()
         TokenKind.String => try self._parseExpression()
         TokenKind.Ident => try self._parseExpression()
         TokenKind.LParen => {
@@ -1391,6 +1395,10 @@ export type Parser {
           self._advance() // consume bool token
           MatchCaseKind.Literal(LiteralAstNode.Bool(value))
         }
+        TokenKind.Char(value) => {
+          self._advance() // consume char token
+          MatchCaseKind.Literal(LiteralAstNode.Char(value))
+        }
         TokenKind.String(value) => {
           self._advance() // consume string token
           MatchCaseKind.Literal(LiteralAstNode.String(value))

diff --git a/projects/compiler/src/test_utils.abra b/projects/compiler/src/test_utils.abra
@@ -12,6 +12,22 @@ export func printTokenAsJson(token: Token, indentLevelStart: Int, currentIndentL
   print("$endIndent}")
 }
 
+export func charToString(charVal: Int): String {
+  // TODO: clean this up
+  val lowestByte = charVal && 0xff
+
+  val higherBytes = [
+    charVal && 0xff00,
+    charVal && 0xff0000,
+    charVal && 0xff000000,
+  ]
+  for b, idx in higherBytes {
+    if b != 0 unreachable("byte $idx of Char($charVal) is not 0")
+  }
+
+  lowestByte.hex()
+}
+
 func printTokenKindAsJson(kind: TokenKind, indentLevelStart: Int, currentIndentLevel: Int) {
   val startIndent = "  ".repeat(indentLevelStart)
   val fieldsIndent = "  ".repeat(currentIndentLevel + 1)
@@ -31,6 +47,12 @@ func printTokenKindAsJson(kind: TokenKind, indentLevelStart: Int, currentIndentL
       println("$fieldsIndent\"name\": \"Bool\",")
       println("$fieldsIndent\"value\": $value")
     }
+    TokenKind.Char(intVal) => {
+      val charAsString = charToString(intVal)
+
+      println("$fieldsIndent\"name\": \"Char\",")
+      println("$fieldsIndent\"value\": \"$charAsString\"")
+    }
     TokenKind.String(value) => {
       println("$fieldsIndent\"name\": \"String\",")
       println("$fieldsIndent\"value\": \"$value\"")
@@ -341,6 +363,11 @@ func printAstNodeKindAsJson(kind: AstNodeKind, indentLevelStart: Int, currentInd
         LiteralAstNode.Int(value) => ("int", value.toString())
         LiteralAstNode.Float(value) => ("float", value.toString())
         LiteralAstNode.Bool(value) => ("bool", value.toString())
+        LiteralAstNode.Char(value) => {
+          val charAsString = charToString(value)
+
+          ("char", "\"$charAsString\"")
+        }
         LiteralAstNode.String(value) => ("string", "\"$value\"")
       }
 
@@ -664,6 +691,10 @@ func printAstNodeKindAsJson(kind: AstNodeKind, indentLevelStart: Int, currentInd
                 LiteralAstNode.Int(value) => ("int", value.toString())
                 LiteralAstNode.Float(value) => ("float", value.toString())
                 LiteralAstNode.Bool(value) => ("bool", value.toString())
+                LiteralAstNode.Char(value) => {
+                  val charAsString = charToString(value)
+                  ("char", "\"$charAsString\"")
+                }
                 LiteralAstNode.String(value) => ("string", "\"$value\"")
               }