feat(web): prototype KMX+ TouchLayout generator (in TS) #12305

jahorton · 2024-08-28T08:49:57Z

In order to build the touch-layout required for OSK construction for KMX+ keyboards, we either need that layout to be pre-compiled... or we need the ability to parse a KMX+ file in order to find the data we need. This PR represents initial efforts toward the latter - parsing the KMX+ format in order to construct a viable TouchLayout that the OSK may consume.

Actual converted layout!

Using the KMX+ keyboard imperial_aramaic.kmx, as output for keymanapp/keyboards#2993...

[
  {
    "id": "default",
    "row": [
      {
        "id": 0,
        "key": [
          {
            "id": "K_BKQUOTE",
            "text": "`",
            "width": 10
          },
          {
            "id": "K_1",
            "text": "𐡘",
            "width": 10
          },
          {
            "id": "K_2",
            "text": "𐡙",
            "width": 10
          },
          {
            "id": "K_3",
            "text": "𐡚",
            "width": 10
          },
          {
            "id": "K_4",
            "text": "𐡛",
            "width": 10
          },
          {
            "id": "K_5",
            "text": "𐡜",
            "width": 10
          },
          {
            "id": "K_6",
            "text": "𐡝",
            "width": 10
          },
          {
            "id": "K_7",
            "text": "𐡞",
            "width": 10
          },
          {
            "id": "K_8",
            "text": "𐡟",
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "id": "K_HYPHEN",
            "text": "-",
            "width": 10
          },
          {
            "id": "K_EQUAL",
            "text": "=",
            "width": 10
          }
        ]
      },
      {
        "id": 1,
        "key": [
          {
            "id": "K_Q",
            "text": "𐡒",
            "width": 10
          },
          {
            "id": "K_W",
            "text": "𐡅",
            "width": 10
          },
          {
            "id": "K_E",
            "text": "𐡄",
            "width": 10
          },
          {
            "id": "K_R",
            "text": "𐡓",
            "width": 10
          },
          {
            "id": "K_T",
            "text": "𐡈",
            "width": 10
          },
          {
            "id": "K_Y",
            "text": "𐡉",
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "id": "K_I",
            "text": "𐡏",
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "id": "K_P",
            "text": "𐡐",
            "width": 10
          },
          {
            "id": "K_LBRKT",
            "text": "[",
            "width": 10
          },
          {
            "id": "K_RBRKT",
            "text": "]",
            "width": 10
          },
          {
            "id": "K_BKSLASH",
            "text": "\\",
            "width": 10
          }
        ]
      },
      {
        "id": 2,
        "key": [
          {
            "id": "K_A",
            "text": "𐡀",
            "width": 10
          },
          {
            "id": "K_S",
            "text": "𐡎",
            "width": 10
          },
          {
            "id": "K_D",
            "text": "𐡃",
            "width": 10
          },
          {
            "id": "K_F",
            "text": "𐡕",
            "width": 10
          },
          {
            "id": "K_G",
            "text": "𐡂",
            "width": 10
          },
          {
            "id": "K_H",
            "text": "𐡇",
            "width": 10
          },
          {
            "id": "K_J",
            "text": "𐡑",
            "width": 10
          },
          {
            "id": "K_K",
            "text": "𐡊",
            "width": 10
          },
          {
            "id": "K_L",
            "text": "𐡋",
            "width": 10
          },
          {
            "id": "K_COLON",
            "text": ";",
            "width": 10
          },
          {
            "id": "K_QUOTE",
            "text": "'",
            "width": 10
          }
        ]
      },
      {
        "id": 3,
        "key": [
          {
            "id": "K_Z",
            "text": "𐡆",
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "id": "K_C",
            "text": "𐡔",
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "id": "K_B",
            "text": "𐡁",
            "width": 10
          },
          {
            "id": "K_N",
            "text": "𐡍",
            "width": 10
          },
          {
            "id": "K_M",
            "text": "𐡌",
            "width": 10
          },
          {
            "id": "K_COMMA",
            "text": ",",
            "width": 10
          },
          {
            "id": "K_PERIOD",
            "text": ".",
            "width": 10
          },
          {
            "id": "K_SLASH",
            "text": "𐡗",
            "width": 10
          }
        ]
      },
      {
        "id": 4,
        "key": [
          {
            "id": "K_SPACE",
            "text": " ",
            "width": 10,
            "layer": "shift"
          }
        ]
      }
    ]
  },
  {
    "id": "shift",
    "row": [
      {
        "id": 0,
        "key": [
          {
            "id": "K_BKQUOTE",
            "text": "~",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_1",
            "text": "!",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_2",
            "text": "@",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_3",
            "text": "#",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_4",
            "text": "$",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_5",
            "text": "%",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_6",
            "text": "^",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_7",
            "text": "&",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_8",
            "text": "*",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_9",
            "text": "(",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_0",
            "text": ")",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_HYPHEN",
            "text": "_",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_EQUAL",
            "text": "+",
            "width": 10,
            "layer": "shift"
          }
        ]
      },
      {
        "id": 1,
        "key": [
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "id": "K_LBRKT",
            "text": "{",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_RBRKT",
            "text": "}",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_BKSLASH",
            "text": "|",
            "width": 10,
            "layer": "shift"
          }
        ]
      },
      {
        "id": 2,
        "key": [
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "id": "K_COLON",
            "text": ":",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_QUOTE",
            "text": "\"",
            "width": 10,
            "layer": "shift"
          }
        ]
      },
      {
        "id": 3,
        "key": [
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "sp": 10,
            "width": 10
          },
          {
            "id": "K_COMMA",
            "text": "<",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_PERIOD",
            "text": ">",
            "width": 10,
            "layer": "shift"
          },
          {
            "id": "K_SLASH",
            "text": "?",
            "width": 10,
            "layer": "shift"
          }
        ]
      },
      {
        "id": 4,
        "key": [
          {
            "id": "K_SPACE",
            "text": " ",
            "width": 10,
            "layer": "shift"
          }
        ]
      }
    ]
  }
]

Known limitations:

I was unable to test this against a keyboard with longpresses, flicks, and/or multitaps at this time.
It does not properly handle key IDs for keys without a "standard" vkey mapping.
- If it maps directly to a K_ key, we handle that perfectly fine.
- If it doesn't, I currently just leave the LDML id for the key, which is... less than ideal.
This layout generator does not currently insert frame keys.
- Note that K_BKSP, K_SHIFT, etc are missing.

keymanapp-test-bot · 2024-08-28T08:50:02Z

User Test Results

Test specification and instructions

ERROR: user tests have not yet been defined

Test Artifacts

Android
Developer
iOS
- Keyman for iOS (simulator image)
- FirstVoices Keyboards for iOS (simulator image)
- TestFlight internal PR build version - 18.0.100 (0.12305.11929)
Keyboards
- Test Keyboards
Web
- KeymanWeb Test Home
Windows

web/src/engine/keyboard/src/lazySpan.ts

web/src/test/auto/headless/engine/keyboard/lazySpan.js

jahorton · 2024-08-29T08:37:29Z

web/src/engine/keyboard/src/kmxPlus/convertLayerList.ts

+
+const keyNames = Object.keys(Codes.keyCodes);
+
+export function convertLayerList(layerList: LayerList, keys: Keys) {


This function converts one layer list (one pre-parsed entry of the layr.lists subtable) into the OSK-friendly TouchLayout format.

jahorton · 2024-08-29T08:39:28Z

web/src/engine/keyboard/src/kmxPlus/layr.ts

+    this.layerLists = layerLists;
+  }
+
+  private processLayers(layerTableOffset: number, strs: Strs, keys: Keys) {


This method may include some "heavy lifting" to assist the convertLayerList method.

jahorton · 2024-08-29T08:45:00Z

web/src/engine/keyboard/src/kmxPlus/ldmlKeyboardObject.ts

+     * - strs
+     * - list - needs strs
+     * - keys - needs lsts
+     * - layr - needs keys, others.


disp could probably go after strs or list - it only really needs to reference strs, I believe.

jahorton · 2024-08-29T08:47:07Z

web/src/engine/keyboard/src/kmxPlus/utils.ts

+  return [...data.slice(start, start+4)].map((x) => String.fromCharCode(x)).join('');
+}
+
+export function decodeString(data: Uint8Array, start: number, len: number) {


For the UTF-16LE strs table.

jahorton · 2024-08-29T08:47:47Z

web/src/test/auto/headless/engine/keyboard/kmx-plus-parsing.spec.js

+
+    const layout = convertLayerList(kbdObj.layr.layerLists[0], kbdObj.keys);
+
+    console.log(JSON.stringify(layout, null, 2));


This line is the source of the example touch-layout I placed in the PR's description.

srl295

reasonable though should be able to reuse constants from the compiler side?

srl295 · 2024-09-01T01:08:24Z

docs/file-formats/kmx-plus-file-format.md

@@ -173,8 +173,8 @@ Then for each string:

 | ∆ | Bits | Name          | Description                                   |
 |---|------|---------------|-----------------------------------------------|
-|16+|  32  | offset        | off: Offset to string                         |
-|20+|  32  | length        | int: Length of string in UTF-16LE code units  |
+|12+|  32  | offset        | off: Offset to string                         |


oops, good catch!

srl295 · 2024-09-01T01:13:55Z

web/src/engine/keyboard/src/kmxPlus/keys.ts

+        }
+      }
+
+      // TODO:  no good current fixture available with data for this.


unit tests only

srl295 · 2024-09-01T01:18:24Z

web/src/engine/keyboard/src/kmxPlus/keys.ts

+    const keyCount    = decodeNumber(rawData,  8);
+    const flicksCount = decodeNumber(rawData, 12);
+
+    const FLICK_GROUP_LEN = 12;


now, shouldn't this file be able to use https://github.com/keymanapp/keyman/blob/master/core/include/ldml/keyman_core_ldml.ts ?
so:

Suggested change

const FLICK_GROUP_LEN = 12;

const FLICK_GROUP_LEN = Constants.length_keys_flick_list;

srl295 · 2024-09-01T01:19:26Z

web/src/engine/keyboard/src/kmxPlus/ldmlKeyboardObject.ts

+     * - strs
+     * - list - needs strs
+     * - keys - needs lsts
+     * - layr - needs keys, others.


mcdurdin · 2024-09-01T01:29:31Z

We are approaching this from a different direction and keeping kmx+ interpretation entirely within Core

srl295 · 2024-09-01T01:54:11Z

We are approaching this from a different direction and keeping kmx+ interpretation entirely within Core

without sharing constants?

don't approve draft pull requests

mcdurdin · 2024-09-01T22:35:18Z

without sharing constants?

A few constants are helpful, e.g. VKs and modifiers but those are not kmx+-specific.

srl295 · 2024-09-02T12:41:20Z

without sharing constants?

A few constants are helpful, e.g. VKs and modifiers but those are not kmx+-specific.

I think you are saying "a different direction from this PR". - my comment above is that https://github.com/keymanapp/keyman/blob/master/core/include/ldml/keyman_core_ldml.ts contains the offsets and KMX+ specific constants from the spec and could be shared here.

Speaking of spec, perhaps the spec corrections should be split out from this PR?

mcdurdin · 2024-09-02T13:27:56Z

Speaking of spec, perhaps the spec corrections should be split out from this PR?

Good point, will do.

Cherry-picked-from: #12305

mcdurdin · 2024-09-02T23:10:30Z

Closing this. We will parse kmx+ layout data in Keyman Core and pass it to Keyman Engine for Web through a new Core API.

feat(web): begin JS-based parsing of KMX+

0ff7934

keymanapp-test-bot bot added the user-test-missing User tests have not yet been defined for the PR label Aug 28, 2024

keymanapp-test-bot bot added this to the A18S9 milestone Aug 28, 2024

github-actions bot added web/ feat labels Aug 28, 2024

jahorton commented Aug 28, 2024

View reviewed changes

feat(web): prototype kmx+ layout spec -> OSK-compatible TouchLayout

2c9a84a

jahorton changed the title ~~feat(web): begin JS-based parsing of KMX+~~ feat(web): prototype KMX+ TouchLayout generator Aug 29, 2024

jahorton changed the title ~~feat(web): prototype KMX+ TouchLayout generator~~ feat(web): prototype KMX+ TouchLayout generator (in TS) Aug 29, 2024

github-actions bot added the docs label Aug 29, 2024

fix(web): modifier 0 => 'default'

1f216be

jahorton commented Aug 29, 2024

View reviewed changes

srl295 self-requested a review August 30, 2024 17:27

darcywong00 modified the milestones: A18S9, A18S10 Aug 31, 2024

srl295 previously approved these changes Sep 1, 2024

View reviewed changes

mcdurdin added a commit that referenced this pull request Sep 2, 2024

fix(common): correct offsets in KMX+ spec

0731c7b

Cherry-picked-from: #12305

mcdurdin mentioned this pull request Sep 2, 2024

fix(common): correct offsets in KMX+ spec #12350

Merged

mcdurdin closed this Sep 2, 2024

mcdurdin deleted the feat/web/initial-kmxplus-parse branch September 2, 2024 23:10

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat(web): prototype KMX+ TouchLayout generator (in TS) #12305

feat(web): prototype KMX+ TouchLayout generator (in TS) #12305

jahorton commented Aug 28, 2024 •

edited

Loading

keymanapp-test-bot bot commented Aug 28, 2024 •

edited

Loading

jahorton Aug 29, 2024

jahorton Aug 29, 2024 •

edited

Loading

jahorton Aug 29, 2024

srl295 Sep 1, 2024

jahorton Aug 29, 2024

jahorton Aug 29, 2024

srl295 left a comment

srl295 Sep 1, 2024

srl295 Sep 1, 2024

srl295 Sep 1, 2024

srl295 Sep 1, 2024

mcdurdin commented Sep 1, 2024

srl295 commented Sep 1, 2024

mcdurdin commented Sep 1, 2024

srl295 commented Sep 2, 2024

mcdurdin commented Sep 2, 2024

mcdurdin commented Sep 2, 2024


		const keyNames = Object.keys(Codes.keyCodes);

		export function convertLayerList(layerList: LayerList, keys: Keys) {


		const layout = convertLayerList(kbdObj.layr.layerLists[0], kbdObj.keys);

		console.log(JSON.stringify(layout, null, 2));

	const FLICK_GROUP_LEN = 12;
	const FLICK_GROUP_LEN = Constants.length_keys_flick_list;

feat(web): prototype KMX+ TouchLayout generator (in TS) #12305

feat(web): prototype KMX+ TouchLayout generator (in TS) #12305

Conversation

jahorton commented Aug 28, 2024 • edited Loading

keymanapp-test-bot bot commented Aug 28, 2024 • edited Loading

User Test Results

Test Artifacts

Choose a reason for hiding this comment

jahorton Aug 29, 2024 • edited Loading

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

srl295 left a comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

mcdurdin commented Sep 1, 2024

srl295 commented Sep 1, 2024

mcdurdin commented Sep 1, 2024

srl295 commented Sep 2, 2024

mcdurdin commented Sep 2, 2024

mcdurdin commented Sep 2, 2024

jahorton commented Aug 28, 2024 •

edited

Loading

keymanapp-test-bot bot commented Aug 28, 2024 •

edited

Loading

jahorton Aug 29, 2024 •

edited

Loading