From f770124c8e1f37b603c5b1f86537c4dbd33214dd Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Wed, 19 Jun 2024 11:16:17 -0300 Subject: [PATCH] Generate reports on private API usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This uses the data gathered in 2dcf4ed to generate some CSV reports about private API usage in the test suite. We’ll use these reports as a starting point for deciding how to remove this private API usage when reusing the test suite as a unified test suite for all our client libraries. Resolves ECO-4834. --- .github/workflows/test-browser.yml | 6 +- .github/workflows/test-node.yml | 6 +- .gitignore | 1 + package-lock.json | 76 +++++++++ package.json | 6 +- scripts/processPrivateApiData/dto.ts | 53 +++++++ scripts/processPrivateApiData/exclusions.ts | 23 +++ scripts/processPrivateApiData/grouping.ts | 76 +++++++++ scripts/processPrivateApiData/load.ts | 16 ++ scripts/processPrivateApiData/output.ts | 144 ++++++++++++++++++ scripts/processPrivateApiData/run.ts | 59 +++++++ .../processPrivateApiData/runtimeContext.ts | 32 ++++ .../processPrivateApiData/staticContext.ts | 117 ++++++++++++++ scripts/processPrivateApiData/utils.ts | 45 ++++++ .../withoutPrivateAPIUsage.ts | 71 +++++++++ 15 files changed, 727 insertions(+), 4 deletions(-) create mode 100644 scripts/processPrivateApiData/dto.ts create mode 100644 scripts/processPrivateApiData/exclusions.ts create mode 100644 scripts/processPrivateApiData/grouping.ts create mode 100644 scripts/processPrivateApiData/load.ts create mode 100644 scripts/processPrivateApiData/output.ts create mode 100644 scripts/processPrivateApiData/run.ts create mode 100644 scripts/processPrivateApiData/runtimeContext.ts create mode 100644 scripts/processPrivateApiData/staticContext.ts create mode 100644 scripts/processPrivateApiData/utils.ts create mode 100644 scripts/processPrivateApiData/withoutPrivateAPIUsage.ts diff --git a/.github/workflows/test-browser.yml b/.github/workflows/test-browser.yml index 5cb471fd1..9bc9360df 100644 --- a/.github/workflows/test-browser.yml +++ b/.github/workflows/test-browser.yml @@ -30,11 +30,15 @@ jobs: - env: PLAYWRIGHT_BROWSER: ${{ matrix.browser }} run: npm run test:playwright + - name: Generate private API usage reports + run: npm run process-private-api-data private-api-usage/*.json - name: Save private API usage data uses: actions/upload-artifact@v4 with: name: private-api-usage-${{ matrix.browser }} - path: private-api-usage + path: | + private-api-usage + private-api-usage-reports - name: Upload test results if: always() uses: ably/test-observability-action@v1 diff --git a/.github/workflows/test-node.yml b/.github/workflows/test-node.yml index db3ba7b7d..d74d0b3db 100644 --- a/.github/workflows/test-node.yml +++ b/.github/workflows/test-node.yml @@ -28,11 +28,15 @@ jobs: - run: npm run test:node env: CI: true + - name: Generate private API usage reports + run: npm run process-private-api-data private-api-usage/*.json - name: Save private API usage data uses: actions/upload-artifact@v4 with: name: private-api-usage-${{ matrix.node-version }} - path: private-api-usage + path: | + private-api-usage + private-api-usage-reports - name: Upload test results if: always() uses: ably/test-observability-action@v1 diff --git a/.gitignore b/.gitignore index 3a3066ac2..f98f1bc7b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ react/ typedoc/generated/ junit/ private-api-usage/ +private-api-usage-reports/ test/support/mocha_junit_reporter/build/ diff --git a/package-lock.json b/package-lock.json index b3ab66145..1a607f821 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,7 @@ "chai": "^4.2.0", "cli-table": "^0.3.11", "cors": "^2.8.5", + "csv": "^6.3.9", "dox": "^1.0.0", "esbuild": "^0.18.10", "esbuild-plugin-umd-wrapper": "ably-forks/esbuild-plugin-umd-wrapper#1.0.7-optional-amd-named-module", @@ -3155,6 +3156,39 @@ "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", "dev": true }, + "node_modules/csv": { + "version": "6.3.9", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.9.tgz", + "integrity": "sha512-eiN+Qu8NwSLxZYia6WzB8xlX/rAQ/8EgK5A4dIF7Bz96mzcr5dW1jlcNmjG0QWySWKfPdCerH3RQ96ZqqsE8cA==", + "dev": true, + "dependencies": { + "csv-generate": "^4.4.1", + "csv-parse": "^5.5.6", + "csv-stringify": "^6.5.0", + "stream-transform": "^3.3.2" + }, + "engines": { + "node": ">= 0.1.90" + } + }, + "node_modules/csv-generate": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.4.1.tgz", + "integrity": "sha512-O/einO0v4zPmXaOV+sYqGa02VkST4GP5GLpWBNHEouIU7pF3kpGf3D0kCCvX82ydIY4EKkOK+R8b1BYsRXravg==", + "dev": true + }, + "node_modules/csv-parse": { + "version": "5.5.6", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.6.tgz", + "integrity": "sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A==", + "dev": true + }, + "node_modules/csv-stringify": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.0.tgz", + "integrity": "sha512-edlXFVKcUx7r8Vx5zQucsuMg4wb/xT6qyz+Sr1vnLrdXqlLD1+UKyWNyZ9zn6mUW1ewmGxrpVwAcChGF0HQ/2Q==", + "dev": true + }, "node_modules/data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -9288,6 +9322,12 @@ "readable-stream": "^3.5.0" } }, + "node_modules/stream-transform": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.2.tgz", + "integrity": "sha512-v64PUnPy9Qw94NGuaEMo+9RHQe4jTBYf+NkTtqkCgeuiNo8NlL0LtLR7fkKWNVFtp3RhIm5Dlxkgm5uz7TDimQ==", + "dev": true + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -13358,6 +13398,36 @@ "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", "dev": true }, + "csv": { + "version": "6.3.9", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.9.tgz", + "integrity": "sha512-eiN+Qu8NwSLxZYia6WzB8xlX/rAQ/8EgK5A4dIF7Bz96mzcr5dW1jlcNmjG0QWySWKfPdCerH3RQ96ZqqsE8cA==", + "dev": true, + "requires": { + "csv-generate": "^4.4.1", + "csv-parse": "^5.5.6", + "csv-stringify": "^6.5.0", + "stream-transform": "^3.3.2" + } + }, + "csv-generate": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.4.1.tgz", + "integrity": "sha512-O/einO0v4zPmXaOV+sYqGa02VkST4GP5GLpWBNHEouIU7pF3kpGf3D0kCCvX82ydIY4EKkOK+R8b1BYsRXravg==", + "dev": true + }, + "csv-parse": { + "version": "5.5.6", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.6.tgz", + "integrity": "sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A==", + "dev": true + }, + "csv-stringify": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.0.tgz", + "integrity": "sha512-edlXFVKcUx7r8Vx5zQucsuMg4wb/xT6qyz+Sr1vnLrdXqlLD1+UKyWNyZ9zn6mUW1ewmGxrpVwAcChGF0HQ/2Q==", + "dev": true + }, "data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -17852,6 +17922,12 @@ "readable-stream": "^3.5.0" } }, + "stream-transform": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.2.tgz", + "integrity": "sha512-v64PUnPy9Qw94NGuaEMo+9RHQe4jTBYf+NkTtqkCgeuiNo8NlL0LtLR7fkKWNVFtp3RhIm5Dlxkgm5uz7TDimQ==", + "dev": true + }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", diff --git a/package.json b/package.json index 03cdacba7..6317e0340 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "chai": "^4.2.0", "cli-table": "^0.3.11", "cors": "^2.8.5", + "csv": "^6.3.9", "dox": "^1.0.0", "esbuild": "^0.18.10", "esbuild-plugin-umd-wrapper": "ably-forks/esbuild-plugin-umd-wrapper#1.0.7-optional-amd-named-module", @@ -155,11 +156,12 @@ "lint": "eslint .", "lint:fix": "eslint --fix .", "prepare": "npm run build", - "format": "prettier --write --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/*.[jt]s docs/**/*.md grunt", - "format:check": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/*.[jt]s docs/**/*.md grunt", + "format": "prettier --write --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/**/*.[jt]s docs/**/*.md grunt", + "format:check": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/**/*.[jt]s docs/**/*.md grunt", "sourcemap": "source-map-explorer build/ably.min.js", "modulereport": "tsc --noEmit --esModuleInterop scripts/moduleReport.ts && esr scripts/moduleReport.ts", "speccoveragereport": "tsc --noEmit --esModuleInterop --target ES2017 --moduleResolution node scripts/specCoverageReport.ts && esr scripts/specCoverageReport.ts", + "process-private-api-data": "tsc --noEmit --esModuleInterop --strictNullChecks scripts/processPrivateApiData/run.ts && esr scripts/processPrivateApiData/run.ts", "docs": "typedoc" } } diff --git a/scripts/processPrivateApiData/dto.ts b/scripts/processPrivateApiData/dto.ts new file mode 100644 index 000000000..acbb272eb --- /dev/null +++ b/scripts/processPrivateApiData/dto.ts @@ -0,0 +1,53 @@ +export type TestPrivateApiContextDto = { + type: 'test'; + title: string; + /** + * null means that either the test isn’t parameterised or that this usage is unique to the specific parameter + */ + parameterisedTestTitle: string | null; + helperStack: string[]; + file: string; + suite: string[]; +}; + +export type HookPrivateApiContextDto = { + type: 'hook'; + title: string; + helperStack: string[]; + file: string; + suite: string[]; +}; + +export type RootHookPrivateApiContextDto = { + type: 'hook'; + title: string; + helperStack: string[]; + file: null; + suite: null; +}; + +export type TestDefinitionPrivateApiContextDto = { + type: 'definition'; + label: string; + helperStack: string[]; + file: string; + suite: string[]; +}; + +export type PrivateApiContextDto = + | TestPrivateApiContextDto + | HookPrivateApiContextDto + | RootHookPrivateApiContextDto + | TestDefinitionPrivateApiContextDto; + +export type PrivateApiUsageDto = { + context: PrivateApiContextDto; + privateAPIIdentifier: string; +}; + +export type TestStartRecord = { + context: TestPrivateApiContextDto; + privateAPIIdentifier: null; +}; + +export type Record = PrivateApiUsageDto | TestStartRecord; diff --git a/scripts/processPrivateApiData/exclusions.ts b/scripts/processPrivateApiData/exclusions.ts new file mode 100644 index 000000000..7578063c4 --- /dev/null +++ b/scripts/processPrivateApiData/exclusions.ts @@ -0,0 +1,23 @@ +import { PrivateApiUsageDto } from './dto'; + +type ExclusionRule = { + privateAPIIdentifier: string; + // i.e. only ignore when called from within this helper + helper?: string; +}; + +/** + * This exclusions mechanism is not currently being used on `main`, but I will use it on a separate unified test suite branch in order to exclude some private API usage that can currently be disregarded in the context of the unified test suite. + */ +export function applyingExclusions(usageDtos: PrivateApiUsageDto[]) { + const exclusionRules: ExclusionRule[] = []; + + return usageDtos.filter( + (usageDto) => + !exclusionRules.some( + (exclusionRule) => + exclusionRule.privateAPIIdentifier === usageDto.privateAPIIdentifier && + (!('helper' in exclusionRule) || usageDto.context.helperStack.includes(exclusionRule.helper!)), + ), + ); +} diff --git a/scripts/processPrivateApiData/grouping.ts b/scripts/processPrivateApiData/grouping.ts new file mode 100644 index 000000000..6359462f6 --- /dev/null +++ b/scripts/processPrivateApiData/grouping.ts @@ -0,0 +1,76 @@ +import { PrivateApiUsageDto } from './dto'; + +export type Group = { + key: Key; + values: Value[]; +}; + +export function grouped( + values: Value[], + keyForValue: (value: Value) => Key, + areKeysEqual: (key1: Key, key2: Key) => boolean, +) { + const result: Group[] = []; + + for (const value of values) { + const key = keyForValue(value); + + let existingGroup = result.find((group) => areKeysEqual(group.key, key)); + + if (existingGroup === undefined) { + existingGroup = { key, values: [] }; + result.push(existingGroup); + } + + existingGroup.values.push(value); + } + + return result; +} + +/** + * Makes sure that each private API is only listed once in a given context. + */ +function dedupeUsages(contextGroups: Group[]) { + for (const contextGroup of contextGroups) { + const newUsages: typeof contextGroup.values = []; + + for (const usage of contextGroup.values) { + const existing = newUsages.find((otherUsage) => otherUsage.privateAPIIdentifier === usage.privateAPIIdentifier); + if (existing === undefined) { + newUsages.push(usage); + } + } + + contextGroup.values = newUsages; + } +} + +export function groupedAndDeduped( + usages: PrivateApiUsageDto[], + keyForUsage: (usage: PrivateApiUsageDto) => Key, + areKeysEqual: (key1: Key, key2: Key) => boolean, +) { + const result = grouped(usages, keyForUsage, areKeysEqual); + dedupeUsages(result); + return result; +} + +/** + * Return value is sorted in decreasing order of usage of a given private API identifer + */ +export function groupedAndSortedByPrivateAPIIdentifier( + groupedByKey: Group[], +): Group[] { + const flattened = groupedByKey.flatMap((group) => group.values.map((value) => ({ key: group.key, value }))); + + const groupedByPrivateAPIIdentifier = grouped( + flattened, + (value) => value.value.privateAPIIdentifier, + (id1, id2) => id1 === id2, + ).map((group) => ({ key: group.key, values: group.values.map((value) => value.key) })); + + groupedByPrivateAPIIdentifier.sort((group1, group2) => group2.values.length - group1.values.length); + + return groupedByPrivateAPIIdentifier; +} diff --git a/scripts/processPrivateApiData/load.ts b/scripts/processPrivateApiData/load.ts new file mode 100644 index 000000000..bfcadc63f --- /dev/null +++ b/scripts/processPrivateApiData/load.ts @@ -0,0 +1,16 @@ +import { readFileSync } from 'fs'; +import { applyingExclusions } from './exclusions'; +import { splittingRecords, stripFilePrefix } from './utils'; +import { Record } from './dto'; + +export function load(jsonFilePath: string) { + let records = JSON.parse(readFileSync(jsonFilePath).toString('utf-8')) as Record[]; + + stripFilePrefix(records); + + let { usageDtos, testStartRecords } = splittingRecords(records); + + usageDtos = applyingExclusions(usageDtos); + + return { usageDtos, testStartRecords }; +} diff --git a/scripts/processPrivateApiData/output.ts b/scripts/processPrivateApiData/output.ts new file mode 100644 index 000000000..18a7b0d34 --- /dev/null +++ b/scripts/processPrivateApiData/output.ts @@ -0,0 +1,144 @@ +import { stringify as csvStringify } from 'csv-stringify/sync'; +import { writeFileSync, existsSync, mkdirSync } from 'fs'; +import { PrivateApiUsageDto, TestStartRecord } from './dto'; +import { Group } from './grouping'; +import { RuntimeContext } from './runtimeContext'; +import { StaticContext } from './staticContext'; +import path from 'path'; + +function suiteAtLevelForCSV(suites: string[] | null, level: number) { + return suites?.[level] ?? ''; +} + +function suitesColumnsForCSV(suites: string[] | null, maxSuiteLevel: number) { + const result: string[] = []; + for (let i = 0; i < maxSuiteLevel; i++) { + result.push(suiteAtLevelForCSV(suites, i)); + } + return result; +} + +function suitesHeaders(maxSuiteLevel: number) { + const result: string[] = []; + for (let i = 0; i < maxSuiteLevel; i++) { + result.push(`Suite (level ${i + 1})`); + } + return result; +} + +function commonHeaders(maxSuiteLevel: number) { + return ['File', ...suitesHeaders(maxSuiteLevel), 'Description'].filter((val) => val !== null) as string[]; +} + +function writeToFile(data: string, name: string) { + const outputDirectoryPath = path.join(__dirname, '..', '..', 'private-api-usage-reports'); + + if (!existsSync(outputDirectoryPath)) { + mkdirSync(outputDirectoryPath); + } + + writeFileSync(path.join(outputDirectoryPath, name), data); +} + +function writeCSVData(rows: string[][], name: string) { + const data = csvStringify(rows); + writeToFile(data, `${name}.csv`); +} + +export function writeRuntimePrivateAPIUsageCSV(contextGroups: Group[]) { + const maxSuiteLevel = Math.max(...contextGroups.map((group) => (group.key.suite ?? []).length)); + + const columnHeaders = [ + 'Context', + ...commonHeaders(maxSuiteLevel), + 'Via parameterised test helper', + 'Via misc. helpers', + 'Private API called', + ]; + + const csvRows = contextGroups + .map((contextGroup) => { + const runtimeContext = contextGroup.key; + + const contextColumns = [ + runtimeContext.type, + runtimeContext.file ?? '', + ...suitesColumnsForCSV(runtimeContext.suite, maxSuiteLevel), + runtimeContext.type === 'definition' ? runtimeContext.label : runtimeContext.title, + ]; + + return contextGroup.values.map((usage) => [ + ...contextColumns, + (usage.context.type === 'test' ? usage.context.parameterisedTestTitle : null) ?? '', + [...usage.context.helperStack].reverse().join(' -> '), + usage.privateAPIIdentifier, + ]); + }) + .flat(); + + writeCSVData([columnHeaders, ...csvRows], 'runtime-private-api-usage'); +} + +export function columnsForStaticContext(staticContext: StaticContext, maxSuiteLevel: number) { + return [ + staticContext.type, + 'file' in staticContext ? staticContext.file : '', + ...suitesColumnsForCSV('suite' in staticContext ? staticContext.suite : null, maxSuiteLevel), + staticContext.title, + ]; +} + +export function writeStaticPrivateAPIUsageCSV(contextGroups: Group[]) { + const maxSuiteLevel = Math.max(...contextGroups.map((group) => ('suite' in group.key ? group.key.suite : []).length)); + + const columnHeaders = ['Context', ...commonHeaders(maxSuiteLevel), 'Private API called']; + + const csvRows = contextGroups + .map((contextGroup) => + contextGroup.values.map((usage) => [ + ...columnsForStaticContext(contextGroup.key, maxSuiteLevel), + usage.privateAPIIdentifier, + ]), + ) + .flat(); + + writeCSVData([columnHeaders, ...csvRows], 'static-private-api-usage'); +} + +export function writeNoPrivateAPIUsageCSV(testStartRecords: TestStartRecord[]) { + const maxSuiteLevel = Math.max(...testStartRecords.map((record) => record.context.suite.length)); + + const columnHeaders = commonHeaders(maxSuiteLevel); + + const csvRows = testStartRecords.map((record) => { + const context = record.context; + return [context.file, ...suitesColumnsForCSV(context.suite, maxSuiteLevel), context.title]; + }); + + writeCSVData([columnHeaders, ...csvRows], 'tests-that-do-not-require-private-api'); +} + +export function writeByPrivateAPIIdentifierCSV(groups: Group[]) { + const maxSuiteLevel = Math.max( + ...groups.flatMap((group) => group.values.map((context) => ('suite' in context ? context.suite.length : 0))), + ); + + const columnHeaders = ['Private API called', 'Context', ...commonHeaders(maxSuiteLevel)]; + + const csvRows = groups + .map((group) => { + const privateAPIIdentifier = group.key; + + return group.values.map((staticContext) => [ + privateAPIIdentifier, + ...columnsForStaticContext(staticContext, maxSuiteLevel), + ]); + }) + .flat(); + + writeCSVData([columnHeaders, ...csvRows], 'static-private-api-usage-by-private-api'); +} + +export function writeSummary(summary: string) { + writeToFile(summary, 'summary.txt'); +} diff --git a/scripts/processPrivateApiData/run.ts b/scripts/processPrivateApiData/run.ts new file mode 100644 index 000000000..814222692 --- /dev/null +++ b/scripts/processPrivateApiData/run.ts @@ -0,0 +1,59 @@ +import { + writeByPrivateAPIIdentifierCSV, + writeNoPrivateAPIUsageCSV, + writeRuntimePrivateAPIUsageCSV, + writeStaticPrivateAPIUsageCSV, + writeSummary, +} from './output'; +import { groupedAndDedupedByRuntimeContext } from './runtimeContext'; +import { groupedAndDedupedByStaticContext } from './staticContext'; +import { load } from './load'; +import { percentageString, sortStaticContextUsages } from './utils'; +import { splitTestsByPrivateAPIUsageRequirement } from './withoutPrivateAPIUsage'; +import { groupedAndSortedByPrivateAPIIdentifier } from './grouping'; + +if (process.argv.length > 3) { + throw new Error('Expected a single argument (path to private API usages JSON file'); +} + +const jsonFilePath = process.argv[2]; + +if (!jsonFilePath) { + throw new Error('Path to private API usages JSON file not specified'); +} + +const { usageDtos, testStartRecords } = load(jsonFilePath); + +const usagesGroupedByRuntimeContext = groupedAndDedupedByRuntimeContext(usageDtos); + +const usagesGroupedByStaticContext = groupedAndDedupedByStaticContext(usageDtos); +sortStaticContextUsages(usagesGroupedByStaticContext); + +const { + requiringPrivateAPIUsage: testsThatRequirePrivateAPIUsage, + notRequiringPrivateAPIUsage: testsThatDoNotRequirePrivateAPIUsage, +} = splitTestsByPrivateAPIUsageRequirement(testStartRecords, usagesGroupedByRuntimeContext); + +const totalNumberOfTests = testStartRecords.length; +const numberOfTestsThatRequirePrivateApiUsage = testsThatRequirePrivateAPIUsage.length; +const numberOfTestsThatDoNotRequirePrivateAPIUsage = testsThatDoNotRequirePrivateAPIUsage.length; + +const summary = `Total number of tests: ${totalNumberOfTests} +Number of tests that require private API usage: ${numberOfTestsThatRequirePrivateApiUsage} (${percentageString( + numberOfTestsThatRequirePrivateApiUsage, + totalNumberOfTests, +)}) +Number of tests that do not require private API usage: ${numberOfTestsThatDoNotRequirePrivateAPIUsage} (${percentageString( + numberOfTestsThatDoNotRequirePrivateAPIUsage, + totalNumberOfTests, +)}) +`; +console.log(summary); + +const byPrivateAPIIdentifier = groupedAndSortedByPrivateAPIIdentifier(usagesGroupedByStaticContext); + +writeRuntimePrivateAPIUsageCSV(usagesGroupedByRuntimeContext); +writeStaticPrivateAPIUsageCSV(usagesGroupedByStaticContext); +writeNoPrivateAPIUsageCSV(testsThatDoNotRequirePrivateAPIUsage); +writeByPrivateAPIIdentifierCSV(byPrivateAPIIdentifier); +writeSummary(summary); diff --git a/scripts/processPrivateApiData/runtimeContext.ts b/scripts/processPrivateApiData/runtimeContext.ts new file mode 100644 index 000000000..25e878b86 --- /dev/null +++ b/scripts/processPrivateApiData/runtimeContext.ts @@ -0,0 +1,32 @@ +import { PrivateApiContextDto, PrivateApiUsageDto } from './dto'; +import { Group, groupedAndDeduped } from './grouping'; +import { joinComponents } from './utils'; + +/** + * Used for determining whether two contexts are equal. + */ +export function runtimeContextIdentifier(context: RuntimeContext) { + return joinComponents([ + { key: 'type', value: context.type }, + { key: 'file', value: context.file ?? 'null' }, + { key: 'suite', value: context.suite?.join(',') ?? 'null' }, + { key: 'title', value: context.type === 'definition' ? context.label : context.title }, + { key: 'helperStack', value: 'helperStack' in context ? context.helperStack.join(',') : 'null' }, + { + key: 'parameterisedTestTitle', + value: ('parameterisedTestTitle' in context ? context.parameterisedTestTitle : null) ?? 'null', + }, + ]); +} + +export type RuntimeContext = PrivateApiContextDto; + +export function groupedAndDedupedByRuntimeContext( + usages: PrivateApiUsageDto[], +): Group[] { + return groupedAndDeduped( + usages, + (usage) => usage.context, + (context1, context2) => runtimeContextIdentifier(context1) === runtimeContextIdentifier(context2), + ); +} diff --git a/scripts/processPrivateApiData/staticContext.ts b/scripts/processPrivateApiData/staticContext.ts new file mode 100644 index 000000000..a3cd6f5ff --- /dev/null +++ b/scripts/processPrivateApiData/staticContext.ts @@ -0,0 +1,117 @@ +import { PrivateApiUsageDto } from './dto'; +import { joinComponents } from './utils'; +import { Group, groupedAndDeduped } from './grouping'; + +export type MiscHelperStaticContext = { + type: 'miscHelper'; + title: string; +}; + +export type TestDefinitionStaticContext = { + type: 'testDefinition'; + title: string; + file: string; + suite: string[]; +}; + +export type RootHookStaticContext = { + type: 'rootHook'; + title: string; +}; + +export type HookStaticContext = { + type: 'hook'; + title: string; + file: string; + suite: string[]; +}; + +export type ParameterisedTestHelperStaticContext = { + type: 'parameterisedTestHelper'; + title: string; + file: string; + suite: string[]; +}; + +export type TestStaticContext = { + type: 'test'; + title: string; + file: string; + suite: string[]; +}; + +export type StaticContext = + | MiscHelperStaticContext + | TestDefinitionStaticContext + | RootHookStaticContext + | HookStaticContext + | ParameterisedTestHelperStaticContext + | TestStaticContext; + +/** + * Used for determining whether two contexts are equal. + */ +export function staticContextIdentifier(context: StaticContext) { + return joinComponents([ + { key: 'type', value: context.type }, + { key: 'file', value: 'file' in context ? context.file : 'null' }, + { key: 'suite', value: 'suite' in context ? context.suite.join(',') : 'null' }, + { key: 'title', value: context.title }, + ]); +} + +export function staticContextForUsage(usage: PrivateApiUsageDto): StaticContext { + if (usage.context.helperStack.length > 0) { + return { + type: 'miscHelper', + title: usage.context.helperStack[0], + }; + } else if (usage.context.type === 'definition') { + return { + type: 'testDefinition', + file: usage.context.file, + suite: usage.context.suite, + title: usage.context.label, + }; + } else if (usage.context.type === 'hook') { + if (usage.context.file === null) { + return { + type: 'rootHook', + title: usage.context.title, + }; + } else { + return { + type: 'hook', + file: usage.context.file, + suite: usage.context.suite, + title: usage.context.title, + }; + } + } else { + if (usage.context.parameterisedTestTitle !== null) { + return { + type: 'parameterisedTestHelper', + file: usage.context.file, + suite: usage.context.suite, + title: usage.context.parameterisedTestTitle, + }; + } else { + return { + type: 'test', + file: usage.context.file, + suite: usage.context.suite, + title: usage.context.title, + }; + } + } +} + +export function groupedAndDedupedByStaticContext( + usages: PrivateApiUsageDto[], +): Group[] { + return groupedAndDeduped( + usages, + (usage) => staticContextForUsage(usage), + (context1, context2) => staticContextIdentifier(context1) === staticContextIdentifier(context2), + ); +} diff --git a/scripts/processPrivateApiData/utils.ts b/scripts/processPrivateApiData/utils.ts new file mode 100644 index 000000000..3e6df542d --- /dev/null +++ b/scripts/processPrivateApiData/utils.ts @@ -0,0 +1,45 @@ +import { PrivateApiUsageDto, Record, TestStartRecord } from './dto'; +import { Group } from './grouping'; +import { StaticContext } from './staticContext'; + +export function stripFilePrefix(records: Record[]) { + for (const record of records) { + if (record.context.file !== null) { + record.context.file = record.context.file.replace('/home/runner/work/ably-js/ably-js/', ''); + } + } +} + +export function splittingRecords(records: Record[]) { + return { + testStartRecords: records.filter((record) => record.privateAPIIdentifier == null) as TestStartRecord[], + usageDtos: records.filter((record) => record.privateAPIIdentifier !== null) as PrivateApiUsageDto[], + }; +} + +export function percentageString(value: number, total: number) { + return `${((100 * value) / total).toLocaleString(undefined, { maximumFractionDigits: 1 })}%`; +} + +/** + * Puts the miscHelper usages (i.e. stuff that doesn’t have file info) first. + */ +export function sortStaticContextUsages(contextGroups: Group[]) { + const original = [...contextGroups]; + + contextGroups.sort((a, b) => { + if (a.key.type === 'miscHelper' && b.key.type !== 'miscHelper') { + return -1; + } + + if (a.key.type !== 'miscHelper' && b.key.type === 'miscHelper') { + return 1; + } + + return original.indexOf(a) - original.indexOf(b); + }); +} + +export function joinComponents(components: { key: string; value: string }[]) { + return components.map((pair) => `${pair.key}=${pair.value}`).join(';'); +} diff --git a/scripts/processPrivateApiData/withoutPrivateAPIUsage.ts b/scripts/processPrivateApiData/withoutPrivateAPIUsage.ts new file mode 100644 index 000000000..2e9d688c5 --- /dev/null +++ b/scripts/processPrivateApiData/withoutPrivateAPIUsage.ts @@ -0,0 +1,71 @@ +import { PrivateApiUsageDto, TestStartRecord } from './dto'; +import { Group } from './grouping'; +import { RuntimeContext } from './runtimeContext'; + +function areStringArraysEqual(arr1: string[], arr2: string[]) { + if (arr1.length !== arr2.length) { + return false; + } + + for (let i = 0; i < arr1.length; i++) { + if (arr1[i] !== arr2[i]) { + return false; + } + } + + return true; +} + +function mustSuiteHierarchyBeExecutedToRunTest(test: TestStartRecord, suites: string[]) { + // i.e. is `suites` a prefix of `test.context.suite`? + return areStringArraysEqual(test.context.suite.slice(0, suites.length), suites); +} + +function mustRuntimeContextBeExecutedToRunTest(test: TestStartRecord, runtimeContext: RuntimeContext) { + if (runtimeContext.type === 'hook') { + if (runtimeContext.file === null) { + // root hook; must be executed to run _any_ test + return true; + } + if ( + runtimeContext.file === test.context.file && + mustSuiteHierarchyBeExecutedToRunTest(test, runtimeContext.suite) + ) { + // the hook must be executed to run this test + return true; + } + } + + // otherwise, return true if and only if it’s the same test + return ( + runtimeContext.type === 'test' && + runtimeContext.file === test.context.file && + areStringArraysEqual(runtimeContext.suite, test.context.suite) && + test.context.title === runtimeContext.title + ); +} + +/** + * This extracts all of the test start records for the tests that can be run without any private API usage. That is, neither the test itself, nor any of the hooks that the test requires, call a private API. It does not consider whether private APIs are required in order to define the test (that is, contexts of type `testDefinition`). + */ +export function splitTestsByPrivateAPIUsageRequirement( + testStartRecords: TestStartRecord[], + groupedUsages: Group[], +): { requiringPrivateAPIUsage: TestStartRecord[]; notRequiringPrivateAPIUsage: TestStartRecord[] } { + const result: { requiringPrivateAPIUsage: TestStartRecord[]; notRequiringPrivateAPIUsage: TestStartRecord[] } = { + requiringPrivateAPIUsage: [], + notRequiringPrivateAPIUsage: [], + }; + + for (const testStartRecord of testStartRecords) { + if ( + groupedUsages.some((contextGroup) => mustRuntimeContextBeExecutedToRunTest(testStartRecord, contextGroup.key)) + ) { + result.requiringPrivateAPIUsage.push(testStartRecord); + } else { + result.notRequiringPrivateAPIUsage.push(testStartRecord); + } + } + + return result; +}