Merge branch 'master' into odata-url-decoding

.circleci/config.yml

@@ -18,7 +18,7 @@ jobs:
             find . -size +1000000c -not -path './.git/*' | grep .
             [[ $? -eq 1 ]]
       - run: make check-file-headers
-      - run: npm ci --legacy-peer-deps
+      - run: npm ci
       - run: node lib/bin/create-docker-databases.js
       - run: make test-ci
       - store_test_results:

.github/workflows/oidc-integration.yml

@@ -28,7 +28,7 @@ jobs:
       with:
         node-version: 20.17.0
         cache: 'npm'
-    - run: npm ci --legacy-peer-deps
+    - run: npm ci
     - run: FAKE_OIDC_ROOT_URL=http://localhost:9898 make fake-oidc-server-ci > fake-oidc-server.log &
     - run: node lib/bin/create-docker-databases.js
     - run: make test-oidc-integration

.github/workflows/s3-e2e.yml

@@ -46,7 +46,7 @@ jobs:
       with:
         node-version: 20.17.0
         cache: 'npm'
-    - run: npm ci --legacy-peer-deps
+    - run: npm ci
     - run: node lib/bin/create-docker-databases.js
     - name: E2E Test
       timeout-minutes: 10

.github/workflows/soak-test.yml

@@ -28,7 +28,7 @@ jobs:
       with:
         node-version: 20.17.0
         cache: 'npm'
-    - run: npm ci --legacy-peer-deps
+    - run: npm ci
     - run: node lib/bin/create-docker-databases.js
     - name: Soak Test
       timeout-minutes: 10

.github/workflows/standard-e2e.yml

@@ -31,7 +31,7 @@ jobs:
       with:
         node-version: 20.10.0
         cache: 'npm'
-    - run: npm ci --legacy-peer-deps
+    - run: npm ci
     - run: node lib/bin/create-docker-databases.js
     - name: E2E Test
       timeout-minutes: 10

.github/workflows/standard-suite.yml

@@ -28,6 +28,6 @@ jobs:
       with:
         node-version: 20.17.0
         cache: 'npm'
-    - run: npm ci --legacy-peer-deps
+    - run: npm ci
     - run: node lib/bin/create-docker-databases.js
     - run: make test

Makefile

@@ -1,7 +1,7 @@
 default: base
 
 node_modules: package.json
-	npm install --legacy-peer-deps
+	npm install
 	touch node_modules
 
 .PHONY: node_version

docs/api.yaml

@@ -1013,7 +1013,7 @@ paths:
       - App User Authentication
       summary: Using App User Authentication
       description: |-
-        To use App User Authentication, first obtain a App User, typically by using the configuration panel in the user interface, or else by using the [App User API Resource](/central-api-accounts-and-users/#app-users). Once you have the token, you can apply it to any eligible action by prefixing the URL with `/key/{appUser}` as follows:
+        To use App User Authentication, first obtain an App User, typically by using the configuration panel in the user interface, or else by using the [App User API Resource](/central-api-accounts-and-users/#app-users). Once you have the token, you can apply it to any eligible action by prefixing the URL with `/key/{appUser}` as follows:
 
         `/v1/key/!Ms7V3$Zdnd63j5HFacIPFEvFAuwNqTUZW$AsVOmaQFf$vIC!F8dJjdgiDnJXXOt/example/request/path`
 
@@ -1045,9 +1045,9 @@ paths:
 
         **Revoking an App User**
 
-        The token associated with a App User is actually just its Session Token. As a result, although a App User Token can uniquely be used as a URL prefix as described here, the session associated with it can be revoked in exactly the same way a session is logged out, by issuing a `DELETE` request to its Session resource.
+        The token associated with an App User is actually just its Session Token. As a result, although an App User Token can uniquely be used as a URL prefix as described here, the session associated with it can be revoked in exactly the same way a session is logged out, by issuing a `DELETE` request to its Session resource.
 
-        Note, however, that a App User cannot revoke itself; a `User` must perform this action.
+        Note, however, that an App User cannot revoke itself; a `User` must perform this action.
       operationId: Revoking an App User
       parameters:
       - name: token
@@ -1875,12 +1875,12 @@ paths:
     delete:
       tags:
       - App Users
-      summary: Deleting a App User
+      summary: Deleting an App User
       description: |-
         You don't have to delete a `App User` in order to cut off its access. Using a `User`'s credentials you can simply [log the App User's session out](/central-api-authentication/#logging-out-current-session) using its token. This will end its session without actually deleting the App User, which allows you to still see it in the configuration panel and inspect its history. This is what the administrative panel does when you choose to "Revoke" the App User.
 
         That said, if you do wish to delete the App User altogether, you can do so by issuing a `DELETE` request to its resource path. App Users cannot delete themselves.
-      operationId: Deleting a App User
+      operationId: Deleting an App User
       parameters:
       - name: id
         in: path
@@ -3670,7 +3670,7 @@ paths:
                 items:
                   $ref: '#/components/schemas/FormAttachment'
               example:
-              - name: myfile.mp3
+              - name: myfile.png
                 type: image
                 exists: true
                 blobExists: true
@@ -4198,7 +4198,7 @@ paths:
                 items:
                   $ref: '#/components/schemas/FormAttachment'
               example:
-              - name: myfile.mp3
+              - name: myfile.png
                 type: image
                 exists: true
                 blobExists: true
@@ -4899,7 +4899,7 @@ paths:
                 items:
                   $ref: '#/components/schemas/FormAttachment'
               example:
-              - name: myfile.mp3
+              - name: myfile.png
                 type: image
                 exists: true
                 blobExists: true
@@ -6165,9 +6165,8 @@ paths:
       tags:
       - Submissions
       summary: Retrieving Submission XML
-      description: To get only the XML of the `Submission` rather than all of the
-        details with the XML as one of many properties, just add `.xml` to the end
-        of the request URL.
+      description: To retrieve the XML of the `Submission`, just append `.xml` to
+        the end of the request URL.
       operationId: Retrieving Submission XML
       parameters:
       - name: projectId
@@ -12264,7 +12263,7 @@ components:
       properties:
         name:
           type: string
-          example: myfile.mp3
+          example: myfile.png
           description: The name of the file as specified in the XForm.
         type:
           $ref: '#/components/schemas/FormAttachmentType'
@@ -12659,7 +12658,7 @@ components:
       properties:
         name:
           type: string
-          example: myfile.mp3
+          example: myfile.png
           description: The name of the file as specified in the Submission XML.
         exists:
           type: boolean

lib/bin/repl.js

@@ -11,9 +11,15 @@ const _ = require('lodash'); // eslint-disable-line import/no-extraneous-depende
 const { sql } = require('slonik');
 
 const container = require('../util/default-container');
+const Problem = require('../util/problem');
 
 (async () => {
-  const context = { ..._.omit(container, 'with'), container, sql };
+  const context = {
+    ..._.omit(container, 'with'),
+    container,
+    sql,
+    Problem,
+  };
   const replGlobals = Object.keys(context);
 
   try {

lib/data/dataset.js

@@ -82,6 +82,14 @@ const getDataset = (xml) =>
     else if (!semverSatisfies(version.get(), '2022.1.0 - 2024.1.x'))
       throw Problem.user.invalidEntityForm({ reason: `Entities specification version [${version.get()}] is not supported.` });
 
+    const warnings = semverSatisfies(version.get(), '>=2024.1.x')
+      ? null
+      : [{
+        type: 'oldEntityVersion',
+        details: { version: version.get() },
+        reason: `Entities specification version [${version.get()}] is not compatible with Offline Entities. Please use version 2024.1.0 or later.`
+      }];
+
     const strippedAttrs = Object.create(null);
     for (const [name, value] of Object.entries(entityAttrs.get()))
       strippedAttrs[stripNamespacesFromPath(name)] = value;
@@ -101,7 +109,7 @@ const getDataset = (xml) =>
     if (actions.length === 0)
       throw Problem.user.invalidEntityForm({ reason: 'The form must specify at least one entity action, for example, create or update.' });
 
-    return Option.of({ name: datasetName, actions });
+    return Option.of({ name: datasetName, actions, warnings });
   });
 
 module.exports = { getDataset, validateDatasetName, validatePropertyName };

lib/data/odata.js

@@ -32,6 +32,7 @@ const hparser = require('htmlparser2');
 const { last, identity } = require('ramda');
 const { SchemaStack } = require('./schema');
 const { shasum } = require('../util/crypto');
+const { url } = require('../util/http');
 const { sanitizeOdataIdentifier } = require('../util/util');
 
 // compares fieldStack to a target tablename and returns whether we are:
@@ -89,7 +90,7 @@ const hashId = (schemaStack, instanceId, slicer = identity) => {
 const navigationLink = (schemaStack, instanceId, slicer = identity) => {
   const fieldStack = slicer(schemaStack.fieldStack);
 
-  const result = [ `Submissions('${encodeURIComponent(instanceId)}')` ];
+  const result = [ url`Submissions('${instanceId}')` ];
   for (let i = 0; i < fieldStack.length; i += 1) {
     const field = fieldStack[i];
     // don't output an ID for the very last repeat, since we want the whole table:

lib/http/endpoint.js

@@ -223,6 +223,12 @@ const defaultResultWriter = (result, request, response, next) => {
 // error thrown upstream that is of our own internal format, this handler does
 // the necessary work to translate that error into an HTTP error and send it out.
 const defaultErrorWriter = (error, request, response) => {
+  if (error instanceof URIError && error.statusCode === 400 && error.status === 400) {
+    // Although there's no way to check definitively, this looks like an
+    // internal error from express caused by decodeURIComponent failing.
+    return defaultErrorWriter(Problem.user.notFound(), request, response);
+  }
+
   if (error?.isProblem === true) {
     // we already have a publicly-consumable error object.
     response.status(error.httpCode).type('application/json').send({

lib/http/preprocessors.js

@@ -31,7 +31,7 @@ const emptyAuthInjector = ({ Auth }, context) => context.with({ auth: Auth.by(nu
 const authHandler = ({ Sessions, Users, Auth }, context) => {
   const authBySessionToken = (token, onFailure = noop) => Sessions.getByBearerToken(token)
     .then((session) => {
-      if (!session.isDefined()) return onFailure();
+      if (session.isEmpty()) return onFailure();
       return context.with({ auth: Auth.by(session.get()) });
     });
 
@@ -114,13 +114,13 @@ const authHandler = ({ Sessions, Users, Auth }, context) => {
 
     // actually try to authenticate with it. no Problem on failure. short circuit
     // out if we have a GET or HEAD request.
-    const maybeSession = authBySessionToken(decodeURIComponent(token));
+    const maybeSession = authBySessionToken(token);
     if ((context.method === 'GET') || (context.method === 'HEAD')) return maybeSession;
 
     // if non-GET run authentication as usual but we'll have to check CSRF afterwards.
     return maybeSession.then((cxt) => { // we have to use cxt rather than context for the linter
       // if authentication failed anyway, just do nothing.
-      if ((cxt == null) || !cxt.auth.session.isDefined()) return;
+      if ((cxt == null) || cxt.auth.session.isEmpty()) return;
 
       const csrf = urlDecode(cxt.body.__csrf);
       if (csrf.isEmpty() || isBlank(csrf.get()) || (cxt.auth.session.get().csrf !== csrf.get())) {

lib/model/frames.js

@@ -113,7 +113,14 @@ Form.Attachment = class extends Frame.define(
   fieldTypes(['int4', 'int4', 'int4', 'int4', 'text', 'text', 'timestamptz'])
 ) {
   forApi() {
-    const data = { name: this.name, type: this.type, exists: (this.blobId != null || this.datasetId != null),  blobExists: this.blobId != null, datasetExists: this.datasetId != null };
+    const data = {
+      name: this.name,
+      type: this.type,
+      hash: this.aux.blob?.md5,
+      exists: (this.blobId != null || this.datasetId != null),
+      blobExists: this.blobId != null,
+      datasetExists: this.datasetId != null
+    };
     if (this.updatedAt != null) data.updatedAt = this.updatedAt;
     return data;
   }

lib/model/frames/entity.js

@@ -128,6 +128,7 @@ Entity.Def.Source = class extends Frame.define(
   table('entity_def_sources', 'source'),
   'details', readable,
   'submissionDefId', 'auditId',
+  'forceProcessed',
   embedded('submissionDef'),
   embedded('audit')
 ) {

lib/model/migrations/20241030-01-add-force-entity-def-source.js

@@ -0,0 +1,20 @@
+// Copyright 2024 ODK Central Developers
+// See the NOTICE file at the top-level directory of this distribution and at
+// https://github.com/getodk/central-backend/blob/master/NOTICE.
+// This file is part of ODK Central. It is subject to the license terms in
+// the LICENSE file found in the top-level directory of this distribution and at
+// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central,
+// including this file, may be copied, modified, propagated, or distributed
+// except according to the terms contained in the LICENSE file.
+
+const up = (db) => db.raw(`
+  ALTER TABLE entity_def_sources
+  ADD COLUMN "forceProcessed" BOOLEAN
+`);
+
+const down = (db) => db.raw(`
+  ALTER TABLE entity_def_sources
+  DROP COLUMN "forceProcessed"
+`);
+
+module.exports = { up, down };

lib/model/query/audits.js

@@ -11,17 +11,19 @@ const { sql } = require('slonik');
 const { map, mergeLeft } = require('ramda');
 const { Actee, Actor, Audit, Dataset, Entity, Form, Project, Submission } = require('../frames');
 const { extender, sqlEquals, page, QueryOptions, unjoiner } = require('../../util/db');
+const { urlDecode } = require('../../util/http');
 const Option = require('../../util/option');
+const Problem = require('../../util/problem');
 const { construct } = require('../../util/util');
 
+const xActionNotes = 'x-action-notes';
 
 const log = (actor, action, actee, details) => ({ run, context }) => {
   const actorId = Option.of(actor).map((x) => x.id).orNull();
   const acteeId = Option.of(actee).map((x) => x.acteeId).orNull();
   const processed = Audit.actionableEvents.includes(action) ? null : sql`clock_timestamp()`;
-  const notes = (context == null) ? null :
-    context.headers['x-action-notes'] == null ? null :
-    decodeURIComponent(context.headers['x-action-notes']); // eslint-disable-line indent
+
+  const notes = Option.of(context?.headers[xActionNotes]).map(val => urlDecode(val).orThrow(Problem.user.invalidHeader(xActionNotes))).orNull();
 
   return run(sql`
 insert into audits ("actorId", action, "acteeId", details, notes, "loggedAt", processed, failures)