feat(playground): Implement codemirror based template string editor

app/package.json

@@ -13,11 +13,14 @@
     "@codemirror/lang-javascript": "^6.2.2",
     "@codemirror/lang-json": "6.0.1",
     "@codemirror/lang-python": "6.1.3",
+    "@codemirror/language": "^6.10.3",
     "@codemirror/lint": "^6.8.1",
     "@codemirror/view": "^6.28.5",
     "@dnd-kit/core": "^6.1.0",
     "@dnd-kit/sortable": "^8.0.0",
     "@dnd-kit/utilities": "^3.2.2",
+    "@lezer/highlight": "^1.2.1",
+    "@lezer/lr": "^1.4.2",
     "@react-three/drei": "^9.108.4",
     "@react-three/fiber": "8.0.12",
     "@tanstack/react-table": "^8.19.3",
@@ -55,6 +58,7 @@
   },
   "devDependencies": {
     "@emotion/react": "^11.11.4",
+    "@lezer/generator": "^1.7.1",
     "@playwright/test": "^1.48.0",
     "@types/d3-format": "^3.0.4",
     "@types/d3-scale-chromatic": "^3.0.3",

app/src/components/templateEditor/TemplateEditor.tsx

@@ -0,0 +1,47 @@
+import React, { useMemo } from "react";
+import { nord } from "@uiw/codemirror-theme-nord";
+import CodeMirror, { ReactCodeMirrorProps } from "@uiw/react-codemirror";
+
+import { useTheme } from "@phoenix/contexts";
+import { assertUnreachable } from "@phoenix/typeUtils";
+
+import { FStringTemplating } from "./language/fStringTemplating";
+import { MustacheLikeTemplating } from "./language/mustacheLikeTemplating";
+
+export const TemplateLanguages = {
+  FString: "f-string", // {variable}
+  Mustache: "mustache", // {{variable}}
+} as const;
+
+type TemplateLanguage =
+  (typeof TemplateLanguages)[keyof typeof TemplateLanguages];
+
+type TemplateEditorProps = ReactCodeMirrorProps & {
+  templateLanguage: TemplateLanguage;
+};
+
+export const TemplateEditor = ({
+  templateLanguage,
+  ...props
+}: TemplateEditorProps) => {
+  const { theme } = useTheme();
+  const codeMirrorTheme = theme === "light" ? undefined : nord;
+  const extensions = useMemo(() => {
+    const ext: TemplateEditorProps["extensions"] = [];
+    switch (templateLanguage) {
+      case TemplateLanguages.FString:
+        ext.push(FStringTemplating());
+        break;
+      case TemplateLanguages.Mustache:
+        ext.push(MustacheLikeTemplating());
+        break;
+      default:
+        assertUnreachable(templateLanguage);
+    }
+    return ext;
+  }, [templateLanguage]);
+
+  return (
+    <CodeMirror theme={codeMirrorTheme} extensions={extensions} {...props} />
+  );
+};

app/src/components/templateEditor/index.tsx

@@ -0,0 +1 @@
+export * from "./TemplateEditor";

app/src/components/templateEditor/language/__tests__/languageUtils.test.ts

@@ -0,0 +1,143 @@
+import { formatFString, FStringTemplatingLanguage } from "../fStringTemplating";
+import { extractVariables } from "../languageUtils";
+import {
+  formatMustacheLike,
+  MustacheLikeTemplatingLanguage,
+} from "../mustacheLikeTemplating";
+
+describe("language utils", () => {
+  it("should extract variable names from a mustache-like template", () => {
+    const tests = [
+      { input: "{{name}}", expected: ["name"] },
+      // TODO: add support for triple mustache escaping or at least use the inner most mustache as value
+      // { input: "{{name}} {{{age}}}", expected: ["name"] },
+      {
+        input:
+          "Hi I'm {{name}} and I'm {{age}} years old and I live in {{city}}",
+        expected: ["name", "age", "city"],
+      },
+      {
+        input: `
+hi there {{name}}
+how are you?
+
+can you help with this json?
+
+{ "name": "John", "age": {{age}} }`,
+        expected: ["name", "age"],
+      },
+    ] as const;
+    tests.forEach(({ input, expected }) => {
+      expect(
+        extractVariables(MustacheLikeTemplatingLanguage.parser, input)
+      ).toEqual(expected);
+    });
+  });
+
+  it("should extract variable names from a f-string template", () => {
+    const tests = [
+      { input: "{name}", expected: ["name"] },
+      { input: "{name} {age}", expected: ["name", "age"] },
+      { input: "{name} {{age}}", expected: ["name"] },
+      {
+        input: "Hi I'm {name} and I'm {age} years old and I live in {city}",
+        expected: ["name", "age", "city"],
+      },
+      {
+        input: `
+hi there {name}
+how are you?
+
+can you help with this json?
+
+{{ "name": "John", "age": {age} }}`,
+        expected: ["name", "age"],
+      },
+    ] as const;
+    tests.forEach(({ input, expected }) => {
+      expect(extractVariables(FStringTemplatingLanguage.parser, input)).toEqual(
+        expected
+      );
+    });
+  });
+
+  it("should format a mustache-like template", () => {
+    const tests = [
+      {
+        input: "{{name}}",
+        variables: { name: "John" },
+        expected: "John",
+      },
+      {
+        input: "{{name}} {{age}}",
+        variables: { name: "John", age: 30 },
+        expected: "John 30",
+      },
+      {
+        input: "{{name}} {age} {{city}}",
+        variables: { name: "John", city: "New York" },
+        expected: "John {age} New York",
+      },
+      {
+        input: `
+hi there {{name}}
+how are you?
+
+can you help with this json?
+
+{ "name": "John", "age": {{age}} }`,
+        variables: { name: "John", age: 30 },
+        expected: `
+hi there John
+how are you?
+
+can you help with this json?
+
+{ "name": "John", "age": 30 }`,
+      },
+    ] as const;
+    tests.forEach(({ input, variables, expected }) => {
+      expect(formatMustacheLike({ text: input, variables })).toEqual(expected);
+    });
+  });
+
+  it("should format a f-string template", () => {
+    const tests = [
+      {
+        input: "{name}",
+        variables: { name: "John" },
+        expected: "John",
+      },
+      {
+        input: "{name} {age}",
+        variables: { name: "John", age: 30 },
+        expected: "John 30",
+      },
+      {
+        input: "{name} {{age}}",
+        variables: { name: "John", age: 30 },
+        expected: "John {age}",
+      },
+      {
+        input: `
+hi there {name}
+how are you?
+
+can you help with this json?
+
+{{ "name": "John", "age": {age} }}`,
+        variables: { name: "John", age: 30 },
+        expected: `
+hi there John
+how are you?
+
+can you help with this json?
+
+{ "name": "John", "age": 30 }`,
+      },
+    ] as const;
+    tests.forEach(({ input, variables, expected }) => {
+      expect(formatFString({ text: input, variables })).toEqual(expected);
+    });
+  });
+});

app/src/components/templateEditor/language/fStringTemplating/fStringTemplating.syntax.grammar

@@ -0,0 +1,25 @@
+@top FStringTemplate {(Template | Text | LEscape | REscape)*}
+
+@skip {} {
+  Template { LBrace Variable+ RBrace }
+}
+
+@local tokens {
+  RBrace { "}" }
+  @else Variable
+}
+
+@skip { space }
+
+@tokens {
+  LEscape { "{{" }
+  REscape { "}}" }
+  LBrace { "{" }
+  Text { ![{}]+ }
+  "}"
+  char { $[\n\r\t\u{20}\u{21}\u{23}-\u{5b}\u{5d}-\u{10ffff}]+ | "\\" esc }
+  esc  { $["\\\/bfnrt] | "u" hex hex hex hex }
+  hex  { $[0-9a-fA-F] }
+  space { @whitespace }
+  @precedence { space, LEscape, REscape, LBrace, Text, char, esc }
+}

app/src/components/templateEditor/language/fStringTemplating/fStringTemplating.syntax.grammar.d.ts

@@ -0,0 +1,3 @@
+import { LRParser } from "@lezer/lr";
+
+export declare const parser: LRParser;

app/src/components/templateEditor/language/fStringTemplating/fStringTemplating.ts

@@ -0,0 +1,63 @@
+import { LanguageSupport, LRLanguage } from "@codemirror/language";
+import { styleTags, tags as t } from "@lezer/highlight";
+
+import { format } from "../languageUtils";
+
+import { parser } from "./fStringTemplating.syntax.grammar";
+
+// https://codemirror.net/examples/lang-package/
+/**
+ * Define the language for the FString templating system
+ *
+ * @example
+ * ```
+ * {question}
+ *
+ * {{
+ *   "answer": {answer}
+ * }}
+ * ```
+ * In this example, the variables are `question` and `answer`.
+ * Double braces are not considered as variables, and will be converted to a single brace on format.
+ */
+export const FStringTemplatingLanguage = LRLanguage.define({
+  parser: parser.configure({
+    props: [
+      // https://lezer.codemirror.net/docs/ref/#highlight.styleTags
+      styleTags({
+        // style the opening brace of a template, not floating braces
+        "Template/LBrace": t.quote,
+        // style the closing brace of a template, not floating braces
+        "Template/RBrace": t.quote,
+        // style variables (stuff inside {})
+        "Template/Variable": t.variableName,
+        // style invalid stuff, undefined tokens will be highlighted
+        "⚠": t.invalid,
+      }),
+    ],
+  }),
+  languageData: {},
+});
+
+export const debugParser = (text: string) => {
+  const tree = FStringTemplatingLanguage.parser.parse(text);
+  return tree.toString();
+};
+
+/**
+ * Formats an FString template with the given variables.
+ */
+export const formatFString = ({
+  text,
+  variables,
+}: Omit<Parameters<typeof format>[0], "parser" | "postFormat">) =>
+  format({
+    parser: FStringTemplatingLanguage.parser,
+    text,
+    variables,
+    postFormat: (text) => text.replaceAll("{{", "{").replaceAll("}}", "}"),
+  });
+
+export function FStringTemplating() {
+  return new LanguageSupport(FStringTemplatingLanguage);
+}

app/src/components/templateEditor/language/fStringTemplating/index.ts

@@ -0,0 +1 @@
+export * from "./fStringTemplating";

app/src/components/templateEditor/language/languageUtils.ts

@@ -0,0 +1,86 @@
+import { LRParser } from "@lezer/lr";
+
+/**
+ * Extracts all variables from a templated string.
+ *
+ * @param parser - The parser for the templating language.
+ *  The parser should be a language parser that emits Variable nodes.
+ * @param text - The text to extract variables from.
+ *
+ * @returns An array of variable names.
+ */
+export const extractVariables = (parser: LRParser, text: string) => {
+  const tree = parser.parse(text);
+  const variables: string[] = [];
+  const cur = tree.cursor();
+  do {
+    if (cur.name === "Variable") {
+      variables.push(text.slice(cur.node.from, cur.node.to));
+    }
+  } while (cur.next());
+  return variables;
+};
+
+/**
+ * Formats a templated string with the given variables.
+ *
+ * The parser should be a language parser that emits Variable nodes as children of some parent node.
+ */
+export const format = ({
+  parser,
+  text,
+  variables,
+  postFormat,
+}: {
+  /**
+   * The parser for the templating language.
+   *
+   * Should be MustacheLikeTemplatingLanguage or FStringTemplatingLanguage.
+   *
+   * format assumes that the language produces a structure where Variable nodes
+   * are children of some parent node, in this case Template.
+   */
+  parser: LRParser;
+  /**
+   * The text to format.
+   */
+  text: string;
+  /**
+   * A mapping of variable names to their values.
+   *
+   * If a variable is not found in this object, it will be left as is.
+   */
+  variables: Record<string, string | number | boolean>;
+  /**
+   * Runs after formatting the text but just before returning the result
+   *
+   * Useful for doing post-parse processing, like replacing double braces with single braces,
+   * or trimming whitespace.
+   */
+  postFormat?: (text: string) => string;
+}) => {
+  if (!text) return "";
+  let result = text;
+  let tree = parser.parse(result);
+  let cur = tree.cursor();
+  do {
+    if (cur.name === "Variable") {
+      // grab the content inside of the braces
+      const variable = result.slice(cur.node.from, cur.node.to);
+      // grab the position of the content including the braces
+      const Template = cur.node.parent!;
+      if (variable in variables) {
+        // replace the content (including braces) with the variable value
+        result = `${result.slice(0, Template.from)}${variables[variable]}${result.slice(Template.to)}`;
+        // reparse the result so that positions are updated
+        tree = parser.parse(result);
+        // reset the cursor to the start of the new tree
+        cur = tree.cursor();
+      }
+    }
+  } while (cur.next());
+  if (postFormat) {
+    result = postFormat(result);
+  }
+  return result;
+};