Generate reports on private API usage

This uses the data gathered in 2dcf4ed to generate some CSV reports
about private API usage in the test suite. We’ll use these reports as a
starting point for deciding how to remove this private API usage when
reusing the test suite as a unified test suite for all our client
libraries.

Resolves ECO-4834.

.github/workflows/test-browser.yml

@@ -30,11 +30,15 @@ jobs:
       - env:
           PLAYWRIGHT_BROWSER: ${{ matrix.browser }}
         run: npm run test:playwright
+      - name: Generate private API usage reports
+        run: npm run process-private-api-data private-api-usage/*.json
       - name: Save private API usage data
         uses: actions/upload-artifact@v4
         with:
           name: private-api-usage-${{ matrix.browser }}
-          path: private-api-usage
+          path: |
+            private-api-usage
+            private-api-usage-reports
       - name: Upload test results
         if: always()
         uses: ably/test-observability-action@v1

.github/workflows/test-node.yml

@@ -28,11 +28,15 @@ jobs:
       - run: npm run test:node
         env:
           CI: true
+      - name: Generate private API usage reports
+        run: npm run process-private-api-data private-api-usage/*.json
       - name: Save private API usage data
         uses: actions/upload-artifact@v4
         with:
           name: private-api-usage-${{ matrix.node-version }}
-          path: private-api-usage
+          path: |
+            private-api-usage
+            private-api-usage-reports
       - name: Upload test results
         if: always()
         uses: ably/test-observability-action@v1

.gitignore

@@ -9,4 +9,5 @@ react/
 typedoc/generated/
 junit/
 private-api-usage/
+private-api-usage-reports/
 test/support/mocha_junit_reporter/build/

package.json

@@ -79,6 +79,7 @@
     "chai": "^4.2.0",
     "cli-table": "^0.3.11",
     "cors": "^2.8.5",
+    "csv": "^6.3.9",
     "dox": "^1.0.0",
     "esbuild": "^0.18.10",
     "esbuild-plugin-umd-wrapper": "ably-forks/esbuild-plugin-umd-wrapper#1.0.7-optional-amd-named-module",
@@ -155,11 +156,12 @@
     "lint": "eslint .",
     "lint:fix": "eslint --fix .",
     "prepare": "npm run build",
-    "format": "prettier --write --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/*.[jt]s docs/**/*.md grunt",
-    "format:check": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/*.[jt]s docs/**/*.md grunt",
+    "format": "prettier --write --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/**/*.[jt]s docs/**/*.md grunt",
+    "format:check": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/**/*.[jt]s docs/**/*.md grunt",
     "sourcemap": "source-map-explorer build/ably.min.js",
     "modulereport": "tsc --noEmit --esModuleInterop scripts/moduleReport.ts && esr scripts/moduleReport.ts",
     "speccoveragereport": "tsc --noEmit --esModuleInterop --target ES2017 --moduleResolution node scripts/specCoverageReport.ts && esr scripts/specCoverageReport.ts",
+    "process-private-api-data": "tsc --noEmit --esModuleInterop --strictNullChecks scripts/processPrivateApiData/run.ts && esr scripts/processPrivateApiData/run.ts",
     "docs": "typedoc"
   }
 }

scripts/processPrivateApiData/dto.ts

@@ -0,0 +1,53 @@
+export type TestPrivateApiContextDto = {
+  type: 'test';
+  title: string;
+  /**
+   * null means that either the test isn’t parameterised or that this usage is unique to the specific parameter
+   */
+  parameterisedTestTitle: string | null;
+  helperStack: string[];
+  file: string;
+  suite: string[];
+};
+
+export type HookPrivateApiContextDto = {
+  type: 'hook';
+  title: string;
+  helperStack: string[];
+  file: string;
+  suite: string[];
+};
+
+export type RootHookPrivateApiContextDto = {
+  type: 'hook';
+  title: string;
+  helperStack: string[];
+  file: null;
+  suite: null;
+};
+
+export type TestDefinitionPrivateApiContextDto = {
+  type: 'definition';
+  label: string;
+  helperStack: string[];
+  file: string;
+  suite: string[];
+};
+
+export type PrivateApiContextDto =
+  | TestPrivateApiContextDto
+  | HookPrivateApiContextDto
+  | RootHookPrivateApiContextDto
+  | TestDefinitionPrivateApiContextDto;
+
+export type PrivateApiUsageDto = {
+  context: PrivateApiContextDto;
+  privateAPIIdentifier: string;
+};
+
+export type TestStartRecord = {
+  context: TestPrivateApiContextDto;
+  privateAPIIdentifier: null;
+};
+
+export type Record = PrivateApiUsageDto | TestStartRecord;

scripts/processPrivateApiData/exclusions.ts

@@ -0,0 +1,23 @@
+import { PrivateApiUsageDto } from './dto';
+
+type ExclusionRule = {
+  privateAPIIdentifier: string;
+  // i.e. only ignore when called from within this helper
+  helper?: string;
+};
+
+/**
+ * This exclusions mechanism is not currently being used on `main`, but I will use it on a separate unified test suite branch in order to exclude some private API usage that can currently be disregarded in the context of the unified test suite.
+ */
+export function applyingExclusions(usageDtos: PrivateApiUsageDto[]) {
+  const exclusionRules: ExclusionRule[] = [];
+
+  return usageDtos.filter(
+    (usageDto) =>
+      !exclusionRules.some(
+        (exclusionRule) =>
+          exclusionRule.privateAPIIdentifier === usageDto.privateAPIIdentifier &&
+          (!('helper' in exclusionRule) || usageDto.context.helperStack.includes(exclusionRule.helper!)),
+      ),
+  );
+}

scripts/processPrivateApiData/grouping.ts

@@ -0,0 +1,76 @@
+import { PrivateApiUsageDto } from './dto';
+
+export type Group<Key, Value> = {
+  key: Key;
+  values: Value[];
+};
+
+export function grouped<Key, Value>(
+  values: Value[],
+  keyForValue: (value: Value) => Key,
+  areKeysEqual: (key1: Key, key2: Key) => boolean,
+) {
+  const result: Group<Key, Value>[] = [];
+
+  for (const value of values) {
+    const key = keyForValue(value);
+
+    let existingGroup = result.find((group) => areKeysEqual(group.key, key));
+
+    if (existingGroup === undefined) {
+      existingGroup = { key, values: [] };
+      result.push(existingGroup);
+    }
+
+    existingGroup.values.push(value);
+  }
+
+  return result;
+}
+
+/**
+ * Makes sure that each private API is only listed once in a given context.
+ */
+function dedupeUsages<Key>(contextGroups: Group<Key, PrivateApiUsageDto>[]) {
+  for (const contextGroup of contextGroups) {
+    const newUsages: typeof contextGroup.values = [];
+
+    for (const usage of contextGroup.values) {
+      const existing = newUsages.find((otherUsage) => otherUsage.privateAPIIdentifier === usage.privateAPIIdentifier);
+      if (existing === undefined) {
+        newUsages.push(usage);
+      }
+    }
+
+    contextGroup.values = newUsages;
+  }
+}
+
+export function groupedAndDeduped<Key>(
+  usages: PrivateApiUsageDto[],
+  keyForUsage: (usage: PrivateApiUsageDto) => Key,
+  areKeysEqual: (key1: Key, key2: Key) => boolean,
+) {
+  const result = grouped(usages, keyForUsage, areKeysEqual);
+  dedupeUsages(result);
+  return result;
+}
+
+/**
+ * Return value is sorted in decreasing order of usage of a given private API identifer
+ */
+export function groupedAndSortedByPrivateAPIIdentifier<Key>(
+  groupedByKey: Group<Key, PrivateApiUsageDto>[],
+): Group<string, Key>[] {
+  const flattened = groupedByKey.flatMap((group) => group.values.map((value) => ({ key: group.key, value })));
+
+  const groupedByPrivateAPIIdentifier = grouped(
+    flattened,
+    (value) => value.value.privateAPIIdentifier,
+    (id1, id2) => id1 === id2,
+  ).map((group) => ({ key: group.key, values: group.values.map((value) => value.key) }));
+
+  groupedByPrivateAPIIdentifier.sort((group1, group2) => group2.values.length - group1.values.length);
+
+  return groupedByPrivateAPIIdentifier;
+}

scripts/processPrivateApiData/load.ts

@@ -0,0 +1,16 @@
+import { readFileSync } from 'fs';
+import { applyingExclusions } from './exclusions';
+import { splittingRecords, stripFilePrefix } from './utils';
+import { Record } from './dto';
+
+export function load(jsonFilePath: string) {
+  let records = JSON.parse(readFileSync(jsonFilePath).toString('utf-8')) as Record[];
+
+  stripFilePrefix(records);
+
+  let { usageDtos, testStartRecords } = splittingRecords(records);
+
+  usageDtos = applyingExclusions(usageDtos);
+
+  return { usageDtos, testStartRecords };
+}