From efdbc3a34138a5389e792de30c6c7e660de66206 Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Fri, 13 Dec 2024 16:59:53 -0800 Subject: [PATCH] Usage metrics for 2024.3 release (#1345) * Beginning to add new usage metrics for 2024.3 release * change defaults to 0 * Additional system metrics * removing entity_create source metrics to put in another pr/commit * Make dataset usage metric faster by pulling properties query out * Add some of the create counts * Count entities created with bulk operation * Fixing a little bug in uploaded blob reporting * remove project has_description * Filter encypted and upgraded form defs --- config/default.json | 2 +- lib/data/analytics.js | 10 +- lib/model/query/analytics.js | 146 ++++++++-- test/integration/other/analytics-queries.js | 297 +++++++++++++++++++- 4 files changed, 423 insertions(+), 32 deletions(-) diff --git a/config/default.json b/config/default.json index 19f31cfca..55b038ebd 100644 --- a/config/default.json +++ b/config/default.json @@ -30,7 +30,7 @@ "analytics": { "url": "https://data.getodk.cloud/v1/key/eOZ7S4bzyUW!g1PF6dIXsnSqktRuewzLTpmc6ipBtRq$LDfIMTUKswCexvE0UwJ9/projects/1/forms/odk-analytics/submissions", "formId": "odk-analytics", - "version": "v2024.2.0_1" + "version": "v2024.3.0_1" }, "s3blobStore": {} } diff --git a/lib/data/analytics.js b/lib/data/analytics.js index 83f9cdbf9..16e63c1c7 100644 --- a/lib/data/analytics.js +++ b/lib/data/analytics.js @@ -31,6 +31,7 @@ const metricsTemplate = { "num_unique_collectors": {}, "database_size": {}, "uses_external_db": 0, + "uses_external_blob_store": 0, "sso_enabled": 0, "num_client_audit_attachments": 0, "num_client_audit_attachments_failures": 0, @@ -41,8 +42,14 @@ const metricsTemplate = { "num_offline_entity_branches": 0, "num_offline_entity_interrupted_branches": 0, "num_offline_entity_submissions_reprocessed": 0, + "num_offline_entity_submissions_force_processed": 0, "max_entity_submission_delay": 0, - "avg_entity_submission_delay": 0 + "avg_entity_submission_delay": 0, + "max_entity_branch_delay": 0, + "num_xml_only_form_defs": 0, + "num_blob_files": 0, + "num_blob_files_on_s3": 0, + "num_reset_failed_to_pending_count": 0 }, "projects": [ { @@ -151,7 +158,6 @@ const metricsTemplate = { } }, "other": { - "has_description": 0, "description_length": 0 }, "datasets": [{ diff --git a/lib/model/query/analytics.js b/lib/model/query/analytics.js index 7adcf676d..ac6fd4dd1 100644 --- a/lib/model/query/analytics.js +++ b/lib/model/query/analytics.js @@ -35,6 +35,12 @@ const databaseExternal = (dbHost) => () => ? 0 : 1); + +const blobStoreExternal = (s3Settings) => () => + (s3Settings && s3Settings.server + ? 1 + : 0); + //////////////////////////////////////////////////////// // SQL QUERIES @@ -139,6 +145,24 @@ WHERE const countClientAuditRows = () => ({ oneFirst }) => oneFirst(sql` SELECT count(*) FROM client_audits`); +const countXmlOnlyFormDefs = () => ({ oneFirst }) => oneFirst(sql` + SELECT count(*) + FROM form_defs + WHERE "xlsBlobId" IS NULL + AND "keyId" IS NULL + AND "version" NOT LIKE '%[upgrade]%'`); + +const countBlobFiles = () => ({ one }) => one(sql` + SELECT + COUNT(*) AS total_blobs, + SUM(CASE WHEN "s3_status" = 'uploaded' THEN 1 ELSE 0 END) AS uploaded_blobs + FROM blobs`); + +const countResetFailedToPending = () => ({ oneFirst }) => oneFirst(sql` + SELECT COUNT(*) + FROM audits + WHERE action = 'blobs.s3.failed-to-pending'`); + // PER PROJECT // Users const countUsersPerRole = () => ({ all }) => all(sql` @@ -397,7 +421,7 @@ group by f."projectId"`); // Datasets const getDatasets = () => ({ all }) => all(sql` SELECT - ds.id, ds."projectId", COUNT(DISTINCT p.id) num_properties, COUNT(DISTINCT e.id) num_entities_total, + ds.id, ds."projectId", COUNT(DISTINCT e.id) num_entities_total, COUNT(DISTINCT CASE WHEN e."createdAt" >= current_date - cast(${DAY_RANGE} as int) THEN e.id END) num_entities_recent, COUNT(DISTINCT CASE WHEN e."updatedAt" IS NOT NULL THEN e.id END) num_entities_updated_total, COUNT(DISTINCT CASE WHEN e."updatedAt" >= current_date - cast(${DAY_RANGE} as int) THEN e.id END) num_entities_updated_recent, @@ -412,9 +436,14 @@ SELECT MAX(COALESCE(updates.update_api_total, 0)) num_entity_updates_api_total, MAX(COALESCE(updates.update_api_recent, 0)) num_entity_updates_api_recent, MAX(COALESCE(conflict_stats.conflicts, 0)) num_entity_conflicts, - MAX(COALESCE(conflict_stats.resolved, 0)) num_entity_conflicts_resolved + MAX(COALESCE(conflict_stats.resolved, 0)) num_entity_conflicts_resolved, + MAX(COALESCE(creates.create_sub_total, 0)) num_entity_create_sub_total, + MAX(COALESCE(creates.create_sub_recent, 0)) num_entity_create_sub_recent, + MAX(COALESCE(creates.create_api_total, 0)) num_entity_create_api_total, + MAX(COALESCE(creates.create_api_recent, 0)) num_entity_create_api_recent, + MAX(COALESCE(bulk_creates.total, 0)) num_entity_create_bulk_total, + MAX(COALESCE(bulk_creates.recent, 0)) num_entity_create_bulk_recent FROM datasets ds - LEFT JOIN ds_properties p ON p."datasetId" = ds.id AND p."publishedAt" IS NOT NULL LEFT JOIN entities e ON e."datasetId" = ds.id LEFT JOIN (dataset_form_defs dfd JOIN form_defs fd ON fd.id = dfd."formDefId" @@ -450,6 +479,33 @@ FROM datasets ds WHERE a."action" = 'entity.update.version' GROUP BY e."datasetId" ) as updates ON ds.id = updates."datasetId" + LEFT JOIN ( + SELECT + COUNT (1) total, + SUM (CASE WHEN a."loggedAt" >= current_date - cast(${DAY_RANGE} as int) THEN 1 ELSE 0 END) recent, + SUM (CASE WHEN a."details"->'submissionDefId' IS NOT NULL THEN 1 ELSE 0 END) create_sub_total, + SUM (CASE WHEN a."details"->'submissionDefId' IS NOT NULL AND a."loggedAt" >= current_date - cast(${DAY_RANGE} as int) + THEN 1 ELSE 0 END) create_sub_recent, + SUM (CASE WHEN a."details"->'submissionDefId' IS NULL THEN 1 ELSE 0 END) create_api_total, + SUM (CASE WHEN a."details"->'submissionDefId' IS NULL AND a."loggedAt" >= current_date - cast(${DAY_RANGE} as int) + THEN 1 ELSE 0 END) create_api_recent, + e."datasetId" + FROM audits a + JOIN entities e on CAST((a.details ->> 'entityId'::TEXT) AS integer) = e.id + WHERE a."action" = 'entity.create' + GROUP BY e."datasetId" + ) as creates ON ds.id = creates."datasetId" + LEFT JOIN ( + SELECT count(1) total, + SUM (CASE WHEN a."loggedAt" >= current_date - cast(${DAY_RANGE} as int) THEN 1 ELSE 0 END) recent, + e."datasetId" + FROM audits a + JOIN entity_def_sources eds on CAST((a.details ->> 'sourceId'::TEXT) AS integer) = eds."id" + JOIN entity_defs ed on ed."sourceId" = eds.id AND root=true + JOIN entities e on ed."entityId" = e.id + WHERE a."action" = 'entity.bulk.create' + GROUP BY e."datasetId" + ) as bulk_creates on ds.id = bulk_creates."datasetId" LEFT JOIN ( SELECT COUNT (1) conflicts, SUM (CASE WHEN e."conflict" IS NULL THEN 1 ELSE 0 END) resolved, @@ -476,6 +532,15 @@ WHERE audits.action = 'entity.bulk.create' GROUP BY ds.id, ds."projectId" `); +const getDatasetProperties = () => ({ all }) => all(sql` +SELECT + ds.id, ds."projectId", COUNT(DISTINCT p.id) num_properties +FROM datasets ds + LEFT JOIN ds_properties p ON p."datasetId" = ds.id AND p."publishedAt" IS NOT NULL +WHERE ds."publishedAt" IS NOT NULL +GROUP BY ds.id, ds."projectId"; +`); + // Offline entities @@ -523,10 +588,13 @@ FROM duplicateRuns; // Number of submissions temporarily held in backlog but were automatically // removed from backlog when preceeding submission came in -const countSubmissionReprocess = () => ({ oneFirst }) => oneFirst(sql` - SELECT COUNT(*) +const countSubmissionBacklogEvents = () => ({ one }) => one(sql` + SELECT + COUNT(CASE WHEN "action" = 'submission.backlog.hold' THEN 1 END) AS "submission.backlog.hold", + COUNT(CASE WHEN "action" = 'submission.backlog.reprocess' THEN 1 END) AS "submission.backlog.reprocess", + COUNT(CASE WHEN "action" = 'submission.backlog.force' THEN 1 END) AS "submission.backlog.force" FROM audits - WHERE "action" = 'submission.backlog.reprocess' + WHERE "action" IN ('submission.backlog.hold', 'submission.backlog.reprocess', 'submission.backlog.force') `); // Measure how much time entities whose source is a submission.create @@ -554,6 +622,21 @@ JOIN submission_defs as sd ON eds."submissionDefId" = sd.id; `); +const measureMaxEntityBranchTime = () => ({ oneFirst }) => oneFirst(sql` + SELECT + COALESCE(MAX(AGE(max_created_at, min_created_at)), '0 seconds'::interval) AS max_time_difference + FROM ( + SELECT + "branchId", + "entityId", + MIN("createdAt") AS min_created_at, + MAX("createdAt") AS max_created_at + FROM entity_defs + GROUP BY "branchId", "entityId" + HAVING "branchId" IS NOT NULL + ) AS subquery; +`); + // Other const getProjectsWithDescriptions = () => ({ all }) => all(sql` select id as "projectId", length(trim(description)) as description_length from projects where coalesce(trim(description),'')!=''`); @@ -575,11 +658,12 @@ const projectMetrics = () => (({ Analytics }) => runSequentially([ Analytics.countSubmissionsByUserType, Analytics.getProjectsWithDescriptions, Analytics.getDatasets, - Analytics.getDatasetEvents + Analytics.getDatasetEvents, + Analytics.getDatasetProperties ]).then(([ userRoles, appUsers, deviceIds, pubLinks, forms, formGeoRepeats, formsEncrypt, formStates, reusedIds, subs, subStates, subEdited, subComments, subUsers, - projWithDesc, datasets, datasetEvents ]) => { + projWithDesc, datasets, datasetEvents, datasetProperties ]) => { const projects = {}; // users @@ -692,9 +776,13 @@ const projectMetrics = () => (({ Analytics }) => runSequentially([ const eventsRow = datasetEvents.find(d => (d.projectId === row.projectId && d.id === row.id)) || { num_bulk_create_events_total: 0, num_bulk_create_events_recent: 0, biggest_bulk_upload: 0 }; + // Properties row + const propertiesRow = datasetProperties.find(d => (d.projectId === row.projectId && d.id === row.id)) || + { num_properties: 0 }; + project.datasets.push({ id: row.id, - num_properties: row.num_properties, + num_properties: propertiesRow.num_properties, num_creation_forms: row.num_creation_forms, num_followup_forms: row.num_followup_forms, num_entities: { total: row.num_entities_total, recent: row.num_entities_recent }, @@ -710,14 +798,18 @@ const projectMetrics = () => (({ Analytics }) => runSequentially([ // 2024.1 metrics num_bulk_create_events: { total: eventsRow.num_bulk_create_events_total, recent: eventsRow.num_bulk_create_events_recent }, - biggest_bulk_upload: eventsRow.biggest_bulk_upload + biggest_bulk_upload: eventsRow.biggest_bulk_upload, + + // 2024.3 metrics + num_entity_creates_sub: { total: row.num_entity_create_sub_total, recent: row.num_entity_create_sub_recent }, + num_entity_creates_api: { total: row.num_entity_create_api_total, recent: row.num_entity_create_api_recent }, + num_entity_creates_bulk: { total: row.num_entity_create_bulk_total, recent: row.num_entity_create_bulk_recent } }); } // other for (const row of projWithDesc) { const project = _getProject(projects, row.projectId); - project.other.has_description = 1; project.other.description_length = row.description_length; } @@ -739,15 +831,19 @@ const previewMetrics = () => (({ Analytics }) => runSequentially([ Analytics.countClientAuditAttachments, Analytics.countClientAuditProcessingFailed, Analytics.countClientAuditRows, + Analytics.countXmlOnlyFormDefs, + Analytics.countBlobFiles, + Analytics.countResetFailedToPending, Analytics.countOfflineBranches, Analytics.countInterruptedBranches, - Analytics.countSubmissionReprocess, + Analytics.countSubmissionBacklogEvents, Analytics.measureEntityProcessingTime, + Analytics.measureMaxEntityBranchTime, Analytics.projectMetrics ]).then(([db, encrypt, bigForm, admins, audits, archived, managers, viewers, collectors, - caAttachments, caFailures, caRows, - oeBranches, oeInterruptedBranches, oeSubReprocess, oeProcessingTime, + caAttachments, caFailures, caRows, xmlDefs, blobFiles, resetFailedToPending, + oeBranches, oeInterruptedBranches, oeBacklogEvents, oeProcessingTime, oeBranchTime, projMetrics]) => { const metrics = clone(metricsTemplate); // system @@ -785,10 +881,20 @@ const previewMetrics = () => (({ Analytics }) => runSequentially([ // 2024.2.0 offline entity metrics metrics.system.num_offline_entity_branches = oeBranches; metrics.system.num_offline_entity_interrupted_branches = oeInterruptedBranches; - metrics.system.num_offline_entity_submissions_reprocessed = oeSubReprocess; + metrics.system.num_offline_entity_submissions_reprocessed = oeBacklogEvents['submission.backlog.reprocess']; + metrics.system.max_entity_submission_delay = oeProcessingTime.max_wait; metrics.system.avg_entity_submission_delay = oeProcessingTime.avg_wait; + // 2024.3.0 offline entity metrics + metrics.system.num_offline_entity_submissions_force_processed = oeBacklogEvents['submission.backlog.force']; + metrics.system.max_entity_branch_delay = oeBranchTime; + metrics.system.num_xml_only_form_defs = xmlDefs; + metrics.system.uses_external_blob_store = Analytics.blobStoreExternal(config.get('default.external.s3blobStore')); + metrics.system.num_blob_files = blobFiles.total_blobs; + metrics.system.num_blob_files_on_s3 = blobFiles.uploaded_blobs; + metrics.system.num_reset_failed_to_pending_count = resetFailedToPending; + return metrics; })); @@ -812,12 +918,16 @@ module.exports = { biggestForm, databaseSize, databaseExternal, + blobStoreExternal, countAdmins, countAppUsers, + countBlobFiles, + countResetFailedToPending, countDeviceIds, countClientAuditAttachments, countClientAuditProcessingFailed, countClientAuditRows, + countXmlOnlyFormDefs, countForms, countFormsEncrypted, countFormFieldTypes, @@ -840,9 +950,11 @@ module.exports = { getLatestAudit, getDatasets, getDatasetEvents, + getDatasetProperties, countOfflineBranches, countInterruptedBranches, - countSubmissionReprocess, + countSubmissionBacklogEvents, measureEntityProcessingTime, - measureElapsedEntityTime + measureElapsedEntityTime, + measureMaxEntityBranchTime }; diff --git a/test/integration/other/analytics-queries.js b/test/integration/other/analytics-queries.js index f495f7bfb..a0f13b692 100644 --- a/test/integration/other/analytics-queries.js +++ b/test/integration/other/analytics-queries.js @@ -213,6 +213,13 @@ describe('analytics task queries', function () { Analytics.databaseExternal(null).should.equal(1); })); + it('should check external blob store configurations', testContainer(async ({ Analytics }) => { + Analytics.blobStoreExternal(null).should.equal(0); + Analytics.blobStoreExternal({}).should.equal(0); + Analytics.blobStoreExternal({ server: 'http://external.store' }).should.equal(1); + Analytics.blobStoreExternal({ server: 'http://external.store', accessKey: 'a', bucketName: 'foo' }).should.equal(1); + })); + describe('counting client audits', () => { it('should count the total number of client audit submission attachments', testService(async (service, { Analytics }) => { const asAlice = await service.login('alice'); @@ -424,6 +431,100 @@ describe('analytics task queries', function () { res.unprocessed.should.equal(1); }); })); + + it('should count form definitions that are XML-only and are not associated with an XLSForm', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + // two existing forms from fixtures + // adds a 3rd form def that does have an associated XLSForm so shouldn't be counted + await asAlice.post('/v1/projects/1/forms') + .send(readFileSync(appRoot + '/test/data/simple.xlsx')) + .set('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') + .set('X-XlsForm-FormId-Fallback', 'testformid') + .expect(200); + + // make another form def that is XML-only + await asAlice.post('/v1/projects/1/forms?publish=true&ignoreWarnings=true') + .send(testData.forms.updateEntity2023) + .set('Content-Type', 'application/xml') + .expect(200); + + let xmlOnlyFormDefs = await container.Analytics.countXmlOnlyFormDefs(); + xmlOnlyFormDefs.should.equal(3); + + // upgrading the old version of the entity form creates more form defs + const { acteeId } = await container.Forms.getByProjectAndXmlFormId(1, 'updateEntity').then(o => o.get()); + await container.Audits.log(null, 'upgrade.process.form.entities_version', { acteeId }); + + // Run form upgrade + await exhaust(container); + + // encrypting the project creates more form defs + await asAlice.post('/v1/projects/1/key') + .send({ passphrase: 'supersecret', hint: 'it is a secret' }) + .expect(200); + + // should count the same number of form defs + xmlOnlyFormDefs = await container.Analytics.countXmlOnlyFormDefs(); + xmlOnlyFormDefs.should.equal(3); + })); + + it('should count the number of binary blob files total and uploaded to external store', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.binaryType) + .expect(200); + + // make 2 blobs from submission attachments + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('xml_submission_file', Buffer.from(testData.instances.binaryType.both), { filename: 'data.xml' }) + .attach('here_is_file2.jpg', Buffer.from('this is test file two'), { filename: 'here_is_file2.jpg' }) + .attach('my_file1.mp4', Buffer.from('this is test file one'), { filename: 'my_file1.mp4' }) + .expect(201); + + // Set upload status on existing blobs + await container.run(sql`update blobs set "s3_status" = 'uploaded' where true`); + + // make a new blob by uploading xlsform + await asAlice.post('/v1/projects/1/forms') + .send(readFileSync(appRoot + '/test/data/simple.xlsx')) + .set('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') + .set('X-XlsForm-FormId-Fallback', 'testformid') + .expect(200); + + const blobs = await container.Analytics.countBlobFiles(); + blobs.should.eql({ total_blobs: 3, uploaded_blobs: 2 }); + })); + + it('should count number of reset failed blob uploads', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.binaryType) + .expect(200); + + // make 2 blobs from submission attachments + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('xml_submission_file', Buffer.from(testData.instances.binaryType.both), { filename: 'data.xml' }) + .attach('here_is_file2.jpg', Buffer.from('this is test file two'), { filename: 'here_is_file2.jpg' }) + .attach('my_file1.mp4', Buffer.from('this is test file one'), { filename: 'my_file1.mp4' }) + .expect(201); + + // set upload status on existing blobs to failed + await container.run(sql`update blobs set "s3_status" = 'failed' where true`); + + // reset failed to pending + await container.Blobs.s3SetFailedToPending(); + + // count events + const count = await container.Analytics.countResetFailedToPending(); + count.should.equal(1); + })); }); describe('user metrics', () => { @@ -944,7 +1045,7 @@ describe('analytics task queries', function () { .replace(/simpleEntity/g, 'simpleEntity2') .replace(/age/g, 'gender'), 1); - const datasets = await container.Analytics.getDatasets(); + const datasets = await container.Analytics.getDatasetProperties(); datasets[0].num_properties.should.be.equal(3); })); @@ -1079,6 +1180,59 @@ describe('analytics task queries', function () { datasets[0].num_entity_updates_recent.should.be.equal(datasets[0].num_entity_updates_api_recent + datasets[0].num_entity_updates_sub_recent); })); + it('should calculate entity creates through different sources (submission, API, bulk', testService(async (service, container) => { + const asAlice = await service.login('alice'); + await createTestForm(service, container, testData.forms.simpleEntity, 1); + + // Create entity via submission + await submitToForm(service, 'alice', 1, 'simpleEntity', testData.instances.simpleEntity.one); + + // Create entity via API + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + label: 'Johnny Doe', + data: { first_name: 'Johnny', age: '22' } + }) + .expect(200); + + // Create 3 entities in bulk + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + source: { name: 'people.csv', size: 100, }, + entities: [ { label: 'a label' }, { label: 'a label' }, { label: 'a label' } ] + }) + .expect(200); + + // let's set date of entity update to long time ago + await container.run(sql`UPDATE audits SET "loggedAt" = '1999-1-1T00:00:00Z' WHERE action in ('entity.create', 'entity.bulk.create')`); + + // Create more recent entities + await submitToForm(service, 'alice', 1, 'simpleEntity', testData.instances.simpleEntity.two); + + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + label: 'foo', + data: { first_name: 'Blaise' } + }) + .expect(200); + + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + source: { name: 'people.csv', size: 100, }, + entities: [ { label: 'a label' }, { label: 'a label' }, { label: 'a label' } ] + }) + .expect(200); + + const datasets = await container.Analytics.getDatasets(); + + datasets[0].num_entity_create_sub_total.should.be.equal(0); + datasets[0].num_entity_create_sub_recent.should.be.equal(0); + datasets[0].num_entity_create_api_total.should.be.equal(2); + datasets[0].num_entity_create_api_recent.should.be.equal(1); + datasets[0].num_entity_create_bulk_total.should.be.equal(6); + datasets[0].num_entity_create_bulk_recent.should.be.equal(3); + })); + it('should calculate number of entities ever updated vs. update actions applied', testService(async (service, container) => { const asAlice = await service.login('alice'); @@ -1580,7 +1734,7 @@ describe('analytics task queries', function () { countInterruptedBranches.should.equal(4); })); - it('should count number of submission.backlog.reprocess events (submissions temporarily in the backlog)', testService(async (service, container) => { + it('should count number of submission.backlog.* events (submissions temporarily in the backlog)', testService(async (service, container) => { await createTestForm(service, container, testData.forms.offlineEntity, 1); const asAlice = await service.login('alice'); @@ -1625,8 +1779,12 @@ describe('analytics task queries', function () { backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); backlogCount.should.equal(0); - let countReprocess = await container.Analytics.countSubmissionReprocess(); - countReprocess.should.equal(1); + let countBacklogEvents = await container.Analytics.countSubmissionBacklogEvents(); + countBacklogEvents.should.eql({ + 'submission.backlog.hold': 1, + 'submission.backlog.reprocess': 1, + 'submission.backlog.force': 0 + }); // Send a future update that will get held in backlog await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') @@ -1643,9 +1801,13 @@ describe('analytics task queries', function () { backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); backlogCount.should.equal(1); - // A submission being put in the backlog is not what is counted so this is still 1 - countReprocess = await container.Analytics.countSubmissionReprocess(); - countReprocess.should.equal(1); + // A submission being put in the backlog is not what is counted so reprocess count is still 1 + countBacklogEvents = await container.Analytics.countSubmissionBacklogEvents(); + countBacklogEvents.should.eql({ + 'submission.backlog.hold': 2, + 'submission.backlog.reprocess': 1, + 'submission.backlog.force': 0 + }); // force processing the backlog await container.Entities.processBacklog(true); @@ -1653,9 +1815,13 @@ describe('analytics task queries', function () { backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); backlogCount.should.equal(0); - // Force processing also doesn't change this count so it is still 1 - countReprocess = await container.Analytics.countSubmissionReprocess(); - countReprocess.should.equal(1); + // Force processing counted now, and reprocessing still only counted once + countBacklogEvents = await container.Analytics.countSubmissionBacklogEvents(); + countBacklogEvents.should.eql({ + 'submission.backlog.hold': 2, + 'submission.backlog.reprocess': 1, + 'submission.backlog.force': 1 + }); //---------- @@ -1689,8 +1855,12 @@ describe('analytics task queries', function () { await exhaust(container); // Two reprocessing events logged now - countReprocess = await container.Analytics.countSubmissionReprocess(); - countReprocess.should.equal(2); + countBacklogEvents = await container.Analytics.countSubmissionBacklogEvents(); + countBacklogEvents.should.eql({ + 'submission.backlog.hold': 3, + 'submission.backlog.reprocess': 2, + 'submission.backlog.force': 1 + }); })); it('should measure time from submission creation to entity version finished processing', testService(async (service, container) => { @@ -1731,6 +1901,69 @@ describe('analytics task queries', function () { waitTime.avg_wait.should.be.greaterThan(0); })); + it('should measure max time between first submission on a branch received and last submission on that branch processed', testService(async (service, container) => { + await createTestForm(service, container, testData.forms.offlineEntity, 1); + const asAlice = await service.login('alice'); + + const emptyTime = await container.Analytics.measureMaxEntityBranchTime(); + emptyTime.should.equal(0); + + // Create entity to update + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + uuid: '12345678-1234-4123-8234-123456789abc', + label: 'label' + }) + .expect(200); + + const branchId = uuid(); + + // Send first update + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('branchId=""', `branchId="${branchId}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + // Send second update + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update2') + .replace('baseVersion="1"', 'baseVersion="2"') + .replace('branchId=""', `branchId="${branchId}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + // Send third update in its own branch + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update3') + .replace('baseVersion="1"', 'baseVersion="3"') + .replace('trunkVersion="1"', 'trunkVersion="3"') + .replace('branchId=""', `branchId="${uuid()}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + // Make another update via the API + await asAlice.patch('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-123456789abc?baseVersion=4') + .send({ label: 'label update' }) + .expect(200); + + const time = await container.Analytics.measureMaxEntityBranchTime(); + time.should.be.greaterThan(0); + + // Set date of entity defs in first branch to 1 day apart + await container.run(sql`UPDATE entity_defs SET "createdAt" = '1999-01-01' WHERE version = 2`); + await container.run(sql`UPDATE entity_defs SET "createdAt" = '1999-01-02' WHERE version = 3`); + const longTime = await container.Analytics.measureMaxEntityBranchTime(); + longTime.should.be.equal(86400); // number of seconds in a day + })); + it('should not see a delay for submissions processed with approvalRequired flag toggled', testService(async (service, container) => { const asAlice = await service.login('alice'); @@ -1952,6 +2185,19 @@ describe('analytics task queries', function () { await exhaust(container); + // sending in an update much later in the chain that will need to be force processed + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update10') + .replace('baseVersion="1"', 'baseVersion="10"') + .replace('branchId=""', `branchId="${branchId}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + await container.Entities.processBacklog(true); + // After the interesting stuff above, encrypt and archive the project // encrypting a project @@ -1984,6 +2230,9 @@ describe('analytics task queries', function () { // can't easily test this metric delete res.system.uses_external_db; delete res.system.sso_enabled; + delete res.system.uses_external_blob_store; + delete res.system.num_blob_files_on_s3; + delete res.system.num_reset_failed_to_pending_count; // everything in system filled in Object.values(res.system).forEach((metric) => @@ -2253,6 +2502,18 @@ describe('analytics task queries', function () { total: 2, recent: 1 }, + num_entity_creates_sub: { + total: 2, + recent: 2 + }, + num_entity_creates_api: { + total: 0, + recent: 0 + }, + num_entity_creates_bulk: { + total: 3, + recent: 3 + }, num_entities_updated: { total: 1, recent: 1 @@ -2290,6 +2551,18 @@ describe('analytics task queries', function () { total: 0, recent: 0 }, + num_entity_creates_sub: { + total: 2, + recent: 2 + }, + num_entity_creates_api: { + total: 0, + recent: 0 + }, + num_entity_creates_bulk: { + total: 0, + recent: 0 + }, num_entities_updated: { total: 0, recent: 0