Merge pull request #9605 from keymanapp/feat/web/gesture-staging-inte…

…gration

feat(web): initial integration of gesture staging 🐵

common/web/gesture-recognizer/src/engine/gestureRecognizer.ts

@@ -11,10 +11,15 @@ export class GestureRecognizer<HoveredItemType> extends TouchpointCoordinator<Ho
   private readonly mouseEngine: MouseEventEngine<HoveredItemType>;
   private readonly touchEngine: TouchEventEngine<HoveredItemType>;
 
-  public constructor(config: GestureRecognizerConfiguration<HoveredItemType>) {
+  public constructor(gestureModelDefinitions: GestureModelDefs<HoveredItemType>, config: GestureRecognizerConfiguration<HoveredItemType>) {
     const preprocessedConfig = preprocessRecognizerConfig(config);
 
-    super();
+    // Possibly just a stop-gap measure... but this provides an empty gesture-spec set definition
+    // that allows testing the path-constrainment functionality without invoking gesture-processing
+    // overhead.
+    gestureModelDefinitions = gestureModelDefinitions || EMPTY_GESTURE_DEFS;
+
+    super(gestureModelDefinitions);
     this.config = preprocessedConfig;
 
     this.mouseEngine = new MouseEventEngine<HoveredItemType>(this.config);

common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts

@@ -53,6 +53,8 @@ export class GestureMatcher<Type> implements PredecessorMatch<Type> {
     }).filter((entry) => !!entry);
   }
 
+  private _isCancelled: boolean = false;
+
   private readonly predecessor?: PredecessorMatch<Type>;
 
   private readonly publishedPromise: ManagedPromise<MatchResult<Type>>; // unsure on the actual typing at the moment.
@@ -147,6 +149,14 @@ export class GestureMatcher<Type> implements PredecessorMatch<Type> {
     }
   }
 
+  public cancel() {
+    this._isCancelled = true;
+  }
+
+  public get isCancelled(): boolean {
+    return this._isCancelled;
+  }
+
   private finalize(matched: boolean, cause: FulfillmentCause) {
     if(this.publishedPromise.isFulfilled) {
       return this._result;
@@ -174,24 +184,6 @@ export class GestureMatcher<Type> implements PredecessorMatch<Type> {
         };
       }
 
-      for(let i = 0; i < this.pathMatchers.length; i++) {
-        const matcher = this.pathMatchers[i];
-        const contactSpec = this.model.contacts[i];
-
-        // If the path already terminated, no need to evaluate further for this contact point.
-        if(matcher.source.isPathComplete) {
-          continue;
-        }
-
-        if(matched && contactSpec.endOnResolve) {
-          matcher.source.terminate(false);
-        } else if(!matched && contactSpec.endOnReject) {
-          // Ending due to gesture-rejection effectively means to cancel the path,
-          // so signal exactly that.
-          matcher.source.terminate(true);
-        }
-      }
-
       // Determine the item source for the item to be reported for this gesture, if any.
       let resolutionItem: Type;
       const itemSource = action.item ?? 'current';
@@ -226,6 +218,38 @@ export class GestureMatcher<Type> implements PredecessorMatch<Type> {
     }
   }
 
+  /**
+   * Applies any source-finalization specified by the model based on whether or not it was matched.
+   * It is invalid to call this method before model evaluation is complete.
+   *
+   * Additionally, this should only be applied for "selected" gesture models - those that "win"
+   * and are accepted as part of a GestureSequence.
+   */
+  public finalizeSources() {
+    if(!this._result) {
+      throw Error("Invalid state for source-finalization - the matcher's evaluation of the gesture model is not yet complete");
+    }
+
+    const matched = this._result.matched;
+    for(let i = 0; i < this.pathMatchers.length; i++) {
+      const matcher = this.pathMatchers[i];
+      const contactSpec = this.model.contacts[i];
+
+      // If the path already terminated, no need to evaluate further for this contact point.
+      if(matcher.source.isPathComplete) {
+        continue;
+      }
+
+      if(matched && contactSpec.endOnResolve) {
+        matcher.source.terminate(false);
+      } else if(!matched && contactSpec.endOnReject) {
+        // Ending due to gesture-rejection effectively means to cancel the path,
+        // so signal exactly that.
+        matcher.source.terminate(true);
+      }
+    }
+  }
+
   /**
    * Determines the active path-matcher best suited to serve as the "primary" path for the gesture.
    *
@@ -312,11 +336,13 @@ export class GestureMatcher<Type> implements PredecessorMatch<Type> {
 
     const contactSpec = this.model.contacts[existingContacts];
     const contactModel = new PathMatcher(contactSpec.model, simpleSource);
+    // Add it early, as we need it to be accessible for reference via .primaryPath stuff below.
+    this.pathMatchers.push(contactModel);
+
     let baseItem: Type = null;
-    if(existingContacts) {
-      // just use the highest-priority item source's base item and call it a day.
-      baseItem = this.primaryPath.baseItem;
-    } else if(this.predecessor && this.model.sustainTimer) {
+    // If there were no existing contacts but a predecessor exists and a sustain timer
+    // has been specified, it needs special base-item handling.
+    if(!existingContacts && this.predecessor && this.model.sustainTimer) {
       const baseItemMode = this.model.sustainTimer.baseItem ?? 'result';
 
       switch(baseItemMode) {
@@ -330,6 +356,10 @@ export class GestureMatcher<Type> implements PredecessorMatch<Type> {
           baseItem = this.predecessor.result.action.item;
           break;
       }
+    } else {
+      // just use the highest-priority item source's base item and call it a day.
+      // There's no need to refer to some previously-existing source for comparison.
+      baseItem = this.primaryPath.baseItem;
     }
 
     if(contactSpec.model.allowsInitialState) {
@@ -340,11 +370,12 @@ export class GestureMatcher<Type> implements PredecessorMatch<Type> {
       }
     }
 
+    // Now that we've done the initial-state check, we can check for instantly-matching path models.
+    contactModel.update();
+
     contactModel.promise.then((resolution) => {
       this.finalize(resolution.type == 'resolve', resolution.cause);
     });
-
-    this.pathMatchers.push(contactModel);
   }
 
   update() {

common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts

@@ -117,6 +117,12 @@ export class GestureSequence<Type> extends EventEmitter<EventMap<Type>> {
     }) ?? [];
 
     if(selection.result.action.type == 'complete' || selection.result.action.type == 'none') {
+      if(this.pushedSelector) {
+        // TODO:  may need extra handling for 'sustain' states - like if a modipress's nested
+        // longpress is in subkey-select mode, to preserve that state instead of interrupting it.
+        this.touchpointCoordinator.popSelector(this.pushedSelector);
+      }
+
       sources.forEach((source) => {
         if(!source.isPathComplete) {
           source.terminate(selection.result.action.type == 'none');

common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts

@@ -233,6 +233,23 @@ export class MatcherSelector<Type> extends EventEmitter<EventMap<Type>> {
         }
       }
 
+      /* If cancellation was requested but not pre-filtered by the synchronizer setup, replace
+       * the result object.  The matcher's Promise may have resolved simultaneously with the
+       * winner but 'lost', a scenario that may require careful handling to clean up.
+       */
+      if(matcher.isCancelled) {
+        result = {
+          matched: false,
+          action: {
+            type: 'none',
+            item: null
+          }
+        };
+      } else {
+        // Since we've selected this matcher, it should apply any model-specified finalization necessary.
+        matcher.finalizeSources();
+      }
+
       // Find ALL associated match-promises for sources matched by the matcher.
       const matchedContactIds = matcher.allSourceIds;
 
@@ -249,6 +266,12 @@ export class MatcherSelector<Type> extends EventEmitter<EventMap<Type>> {
         // It's already been handled; do not re-attempt.
         return;
       }
+
+      if(matcher.isCancelled) {
+        // Fortunately, the rest of the code will help us recover from the state.
+        console.warn("Unexpected state:  a cancelled GestureMatcher was still listed as a possibility");
+      }
+
       this.potentialMatchers.splice(matcherIndex, 1);
 
       /*
@@ -331,29 +354,36 @@ export class MatcherSelector<Type> extends EventEmitter<EventMap<Type>> {
             return !losingMatchers.find((matcher2) => matcher == matcher2);
           });
 
-          // Drop all trackers for the matched sources.
+          /*
+          * While the 'synchronizer' setup will perfectly handle most cases, we need this block to catch
+          * a somewhat niche case:  if a second source was added to the matcher at a later point in time,
+          * there are two separate Promise handlers - with separate synchronization sets.  We use the
+          * `cancel` method to ensure that cancellation from one set propagates to the other handler.
+          * (It seems the simplest & most straightforward approach to do ensure localized, per-matcher
+          * consistency without mangling any matchers that shouldn't be affected.)
+          *
+          * This can arise if a modipress is triggered at the same time a new touchpoint begins, which
+          * could trigger a simple-tap.
+          */
+          losingMatchers.forEach((matcher) => {
+            matcher.cancel();
+          });
+
+          // Drop the newly-cancelled trackers.
           this._sourceSelector = this._sourceSelector.filter((a) => !sourceMetadata.find((b) => a == b));
 
           // And now for one line with some "heavy lifting":
 
           /*
-           * Does two things:
-           * 1. Fulfills the contract set by `matchGesture`.
+           * Fulfills the contract set by `matchGesture`.
            *
-           * 2. Fulfilling the ManagedPromise acts as a synchronizer, facilitating the guarantee at
-           * the start of this closure.  It's set synchronously, so other gesture-matchers that
-           * call into this method will know that a match has already fulfilled for the matched
+           * Also, fulfilling the ManagedPromise acts as a synchronizer, partially facilitating the
+           * guarantee at the start of this closure.  It's set synchronously, so other gesture-matchers
+           * that call into this method will know that a match has already fulfilled for the matched
            * source(s).  Any further matchers will be silently ignored, effectively cancelling them.
-           *
-           * If we're within this closure, the closure's synchronizer-promise matches the instance
-           * currently set on its `tracker` - as are any others affected by the resolving matcher.
-           *
-           * It _is_ possible that we may need to resolve a Promise not included in the synchronizer
-           * set - if a second contact / source was added at a later point in time to something that
-           * started single-contact.  Two separate 'raise' attempts would occur, since the links to
-           * this method were set for each source independently.  The most consistent way to ensure
-           * synchronization is thus to rely on the instance annotated on the tracker itself for
-           * each matched source.
+           * However, this fails to handle the case where two separate calls to matcherSelectionFilter
+           * occur for the same matcher due to one source being added at a later point in time;
+           * this is what the `cancel`
            */
           tracker.matchPromise.resolve({matcher, result});
         }

common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts

@@ -2,6 +2,8 @@ import EventEmitter from "eventemitter3";
 import { InputEngineBase } from "./inputEngineBase.js";
 import { GestureSource, GestureSourceSubview } from "./gestureSource.js";
 import { MatcherSelector } from "./gestures/matchers/matcherSelector.js";
+import { GestureSequence } from "./gestures/matchers/gestureSequence.js";
+import { GestureModelDefs, getGestureModelSet } from "./gestures/specs/gestureModelDefs.js";
 
 interface EventMap<HoveredItemType> {
   /**
@@ -10,6 +12,8 @@ interface EventMap<HoveredItemType> {
    * @returns
    */
   'inputstart': (input: GestureSource<HoveredItemType>) => void;
+
+  'recognizedgesture': (sequence: GestureSequence<HoveredItemType>) => void;
 }
 
 /**
@@ -24,11 +28,21 @@ export class TouchpointCoordinator<HoveredItemType> extends EventEmitter<EventMa
   private inputEngines: InputEngineBase<HoveredItemType>[];
   private selectorStack: MatcherSelector<HoveredItemType>[] = [new MatcherSelector()];
 
+  private gestureModelDefinitions: GestureModelDefs<HoveredItemType>;
+
   private _activeSources: GestureSource<HoveredItemType>[] = [];
+  private _activeGestures: GestureSequence<HoveredItemType>[] = [];
 
-  public constructor() {
+  public constructor(gestureModelDefinitions: GestureModelDefs<HoveredItemType>, inputEngines?: InputEngineBase<HoveredItemType>[]) {
     super();
+
+    this.gestureModelDefinitions = gestureModelDefinitions;
     this.inputEngines = [];
+    if(inputEngines) {
+      for(let engine of inputEngines) {
+        this.addEngine(engine);
+      }
+    }
   }
 
   public pushSelector(selector: MatcherSelector<HoveredItemType>) {
@@ -58,29 +72,59 @@ export class TouchpointCoordinator<HoveredItemType> extends EventEmitter<EventMa
 
   private readonly onNewTrackedPath = (touchpoint: GestureSource<HoveredItemType>) => {
     this.addSimpleSourceHooks(touchpoint);
-
-    // ... stuff
+    const modelDefs = this.gestureModelDefinitions;
+    const selector = this.currentSelector;
+
+    const firstSelectionPromise = selector.matchGesture(touchpoint, getGestureModelSet(modelDefs, selector.baseGestureSetId));
+    firstSelectionPromise.then((selection) => {
+      if(selection.result.matched == false) {
+        return;
+      }
+
+      // For multitouch gestures, only report the gesture **once**.
+      const sourceIDs = selection.matcher.allSourceIds;
+      for(let sequence of this._activeGestures) {
+        if(!!sequence.allSourceIds.find((id1) => !!sourceIDs.find((id2) => id1 == id2))) {
+          // We've already established (and thus, already reported) a GestureSequence for this selection.
+          return;
+        }
+      }
+
+      const gestureSequence = new GestureSequence(selection, modelDefs, this.currentSelector, this);
+      this._activeGestures.push(gestureSequence);
+      gestureSequence.on('complete', () => {
+        // When the GestureSequence is fully complete and all related `firstSelectionPromise`s have
+        // had the chance to resolve, drop the reference; prevent memory leakage.
+        const index = this._activeGestures.indexOf(gestureSequence);
+        if(index != -1) {
+          this._activeGestures.splice(index, 1);
+        }
+      });
+
+      // Could track sequences easily enough; the question is how to tell when to 'let go'.
+
+      this.emit('recognizedgesture', gestureSequence);
+
+      // Any related 'push' mechanics that may still be lingering are currently handled by GestureSequence
+      // during its 'completion' processing.  (See `GestureSequence.selectionHandler`.)
+    });
 
     this.emit('inputstart', touchpoint);
   }
 
-  private doGestureUpdate(source: GestureSource<HoveredItemType>) {
-    // Should probably ensure data-updates for multi-contact gestures are synchronized
-    // before proceeding.  Single-contact cases are inherently synchronized, of course.
-    //
-    // Should a gesture type have geometric requirements on the current location of active
-    // touchpaths, having a desync during a quick movement could cause the calculated
-    // distance between the locations to be markedly different than expected.
-
-    // TODO: stuff, including synchronization.  Probably do that on the caller,
-    // rather than here?
+  public get activeGestures(): GestureSequence<HoveredItemType>[] {
+    return [].concat(this._activeGestures);
   }
 
   private addSimpleSourceHooks(touchpoint: GestureSource<HoveredItemType>) {
-    touchpoint.path.on('step', () => this.doGestureUpdate(touchpoint));
 
     touchpoint.path.on('invalidated', () => {
-      // TODO: on cancellation, is there any other cleanup to be done?
+      // GestureSequence _should_ handle any other cleanup internally as fallout
+      // from the path being cancelled.
+
+      // To consider: should it specially mark if it 'completed' due to cancellation,
+      // or is that safe to infer from the tracked GestureSource(s)?
+      // Currently, we're going with the latter.
 
       // Also mark the touchpoint as no longer active.
       let i = this._activeSources.indexOf(touchpoint);