Fix observation sync issues

plugins/arcgis/service/src/FeatureLayerProcessor.ts

@@ -85,7 +85,7 @@ export class FeatureLayerProcessor {
 
         for (const arcObservation of observations.deletions) {
             if (this.layerInfo.geometryType == arcObservation.esriGeometryType) {
-                this.sender.sendDelete(Number(arcObservation.id));
+                this.sender.sendDelete(arcObservation.id);
             }
         }
     }

plugins/arcgis/service/src/FeatureQuerier.ts

@@ -1,7 +1,8 @@
 import { ArcGISPluginConfig } from "./ArcGISPluginConfig";
 import { LayerInfo } from "./LayerInfo";
 import { QueryObjectResult } from "./QueryObjectResult";
-import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request";
+import { ArcGISIdentityManager } from "@esri/arcgis-rest-request";
+import { queryFeatures } from '@esri/arcgis-rest-feature-service';
 
 /**
  * Performs various queries on observations for a specific arc feature layer.
@@ -52,22 +53,17 @@ export class FeatureQuerier {
      * @param geometry query the geometry, default is true
      */
     async queryObservation(observationId: string, response: (result: QueryObjectResult) => void, fields?: string[], geometry?: boolean) {
-        const queryUrl = new URL(this._url)
-        if (this._config.eventIdField == null) {
-            queryUrl.searchParams.set('where', `${this._config.observationIdField} LIKE '${observationId}${this._config.idSeparator}%'`);
-        } else {
-            queryUrl.searchParams.set('where', `${this._config.observationIdField} = ${observationId}`);
-        }
-        queryUrl.searchParams.set('outFields', this.outFields(fields))
-        queryUrl.searchParams.set('returnGeometry', geometry === false ? 'false' : 'true')
-        this._console.info('ArcGIS query: ' + queryUrl)
-
-        const queryResponse = await request(queryUrl.toString(), {
+        const where = !this._config.eventIdField 
+            ? `${this._config.observationIdField} LIKE '${observationId}${this._config.idSeparator}%'`
+            : `${this._config.observationIdField} = '${observationId}'`;
+        this._console.info('ArcGIS query observation: ' + this._url.toString() + where);
+        await queryFeatures({
+            url: this._url.toString(),
             authentication: this._identityManager,
-            params: { f: 'json' }
-        });
-
-        response(queryResponse as QueryObjectResult);
+            where,
+            returnGeometry: geometry,
+            outFields: fields?.length ? fields : '*'
+        }).then((queryResponse) => response(queryResponse as QueryObjectResult)).catch((error) => this._console.error('Error in FeatureQuerier.queryObservation :: ' + error));
     }
 
     /**
@@ -77,19 +73,14 @@ export class FeatureQuerier {
      * @param geometry query the geometry, default is true
      */
     async queryObservations(response: (result: QueryObjectResult) => void, fields?: string[], geometry?: boolean) {
-        const queryUrl = new URL(this._url)
-        queryUrl.searchParams.set('where', `${this._config.observationIdField} IS NOT NULL`);
-        queryUrl.searchParams.set('outFields', this.outFields(fields));
-        queryUrl.searchParams.set('returnGeometry', geometry === false ? 'false' : 'true');
-
-        this._console.info('ArcGIS query: ' + queryUrl)
-
-        const queryResponse = await request(queryUrl.toString(), {
+        this._console.info('ArcGIS query observation: ' + this._url.toString());
+        await queryFeatures({
+            url: this._url.toString(),
             authentication: this._identityManager,
-            params: { f: 'json' }
-        });
-
-        response(queryResponse as QueryObjectResult);
+            where: `${this._config.observationIdField} IS NOT NULL`,
+            returnGeometry: geometry,
+            outFields: fields?.length ? fields : '*'
+        }).then((queryResponse) => response(queryResponse as QueryObjectResult)).catch((error) => this._console.error('Error in FeatureQuerier.queryObservations :: ' + error));
     }
 
     /**
@@ -98,37 +89,14 @@ export class FeatureQuerier {
      * @param field field to query
      */
     async queryDistinct(response: (result: QueryObjectResult) => void, field: string) {
-        const queryUrl = new URL(this._url);
-        queryUrl.searchParams.set('where', `${field} IS NOT NULL`);
-        queryUrl.searchParams.set('returnDistinctValues', 'true');
-        queryUrl.searchParams.set('outFields', this.outFields([field]));
-        queryUrl.searchParams.set('returnGeometry', 'false');      
-        this._console.info('ArcGIS query: ' + queryUrl)
-
-        try {
-            const queryResponse = await request(queryUrl.toString(), {
-                authentication: this._identityManager,
-                params: { f: 'json' }
-
-            });  
-
-            response(queryResponse as QueryObjectResult);
-        } catch (err) {
-            console.error("could not query", err)
-        }
-    }
-
-    /**
-     * Build the out fields query parameter
-     * @param fields query fields
-     * @returns out fields
-     */
-    private outFields(fields?: string[]): string {
-        if (fields != null && fields.length > 0) {
-            return fields.join(',');
-        } else {
-            return '*';
-        }
+        this._console.info('ArcGIS query observation: ' + this._url.toString());
+        await queryFeatures({
+            url: this._url.toString(),
+            authentication: this._identityManager,
+            where: `${field} IS NOT NULL`,
+            returnGeometry: false,
+            outFields: field ? [field] : '*',
+            returnDistinctValues: true
+        }).then((queryResponse) => response(queryResponse as QueryObjectResult)).catch((error) => this._console.error('Error in FeatureQuerier.queryDistinct :: ' + error));
     }
-
 }

plugins/arcgis/service/src/ObservationBinner.ts

@@ -89,7 +89,8 @@ export class ObservationBinner {
         const bins = new ObservationBins();
 
         for (const arcObservation of observations.observations) {
-            if (arcObservation.lastModified != arcObservation.createdAt) {
+            // TODO: Would probably want a better way to determine which observations need to be updated in arcgis
+            if (observations.firstRun || arcObservation.lastModified != arcObservation.createdAt) {
                 bins.updates.add(arcObservation);
             } else if (!this._addedObs.has(arcObservation.id)) {
                 bins.adds.add(arcObservation);

plugins/arcgis/service/src/ObservationProcessor.ts

@@ -410,7 +410,7 @@ export class ObservationProcessor {
 			}
 			arcObjects.firstRun = this._firstRun;
 			for (const layerProcessor of layerProcessors) {
-				layerProcessor.processArcObjects(JSON.parse(JSON.stringify(arcObjects)));
+				layerProcessor.processArcObjects(arcObjects);
 			}
 			newNumberLeft -= latestObs.items.length;
 			pagingSettings.pageIndex++;

plugins/arcgis/service/src/ObservationsSender.ts

@@ -7,9 +7,8 @@ import { AttachmentInfosResult, AttachmentInfo } from './AttachmentInfosResult';
 import environment from '@ngageoint/mage.service/lib/environment/env'
 import fs from 'fs'
 import path from 'path'
-import FormData from 'form-data';
 import { ArcGISIdentityManager, IFeature, request } from "@esri/arcgis-rest-request"
-import { addFeatures, updateFeatures, deleteFeatures, getAttachments, updateAttachment, addAttachment, deleteAttachments } from "@esri/arcgis-rest-feature-service";
+import { addFeatures, updateFeatures, getAttachments, updateAttachment, addAttachment, deleteAttachments } from "@esri/arcgis-rest-feature-service";
 
 /**
  * Class that transforms observations into a json string that can then be sent to an arcgis server.
@@ -21,16 +20,6 @@ export class ObservationsSender {
      */
     private _url: string;
 
-    /**
-     * The full url to the feature layer receiving observations.
-     */
-    private _urlAdd: string;
-
-    /**
-     * The full url to the feature layer receiving updates.
-     */
-    private _urlUpdate: string;
-
     /**
      * Used to log to the console.
      */
@@ -56,8 +45,6 @@ export class ObservationsSender {
      */
     constructor(layerInfo: LayerInfo, config: ArcGISPluginConfig, identityManager: ArcGISIdentityManager, console: Console) {
         this._url = layerInfo.url;
-        this._urlAdd = this._url + '/addFeatures';
-        this._urlUpdate = this._url + '/updateFeatures';
         this._console = console;
         this._attachmentDirectory = environment.attachmentBaseDirectory;
         this._config = config;
@@ -70,14 +57,13 @@ export class ObservationsSender {
      * @param observations The observations to convert.
      */
     sendAdds(observations: ArcObjects) {
-        this._console.info('ArcGIS addFeatures url ' + this._urlAdd);
+        this._console.info('ArcGIS addFeatures');
 
-        let responseHandler = this.addResponseHandler(observations);
         addFeatures({
             url: this._url,
             authentication: this._identityManager,
             features: observations.objects as IFeature[]
-        }).then(responseHandler).catch((error) => this._console.error(error));
+        }).then(this.responseHandler(observations)).catch((error) => this._console.error(error));
     }
 
     /**
@@ -87,68 +73,47 @@ export class ObservationsSender {
      * @returns The json string of the observations.
      */
     sendUpdates(observations: ArcObjects) {
-        this._console.info('ArcGIS updateFeatures url ' + this._urlUpdate);
+        this._console.info('ArcGIS updateFeatures');
 
-        let responseHandler = this.updateResponseHandler(observations);
         updateFeatures({
             url: this._url,
             authentication: this._identityManager,
             features: observations.objects as IFeature[]
-        }).then(responseHandler).catch((error) => this._console.error(error));
+        }).then(this.responseHandler(observations, true)).catch((error) => this._console.error(error));
     }
 
     /**
      * Delete an observation.
      * @param id The observation id.
      */
-    sendDelete(id: number) {
-        const url = this._url + '/deleteFeatures'
-        this._console.info('ArcGIS deleteFeatures url ' + url + ', ' + this._config.observationIdField + ': ' + id)
+    sendDelete(id: string) {
+        this._console.info('ArcGIS deleteFeatures id: ' + id)
 
-        deleteFeatures({
-            url,
+        request(`${this._url}/deleteFeatures`, {
+            httpMethod: 'POST',
             authentication: this._identityManager,
-            objectIds: [id]
-        }).catch((error) => this._console.error(error));
+            params: {
+                where: `${this._config.observationIdField} LIKE \'%${id}\'`
+            }
+        }).catch((error) => this._console.error('Error in ObservationSender.sendDelete :: ' + error));
     }
 
     /**
      * Deletes all observations that are apart of a specified event.
      * @param id The event id.
      */
     sendDeleteEvent(id: number) {
+        this._console.info('ArcGIS deleteFeatures by event ' + this._config.observationIdField + ': ' + id);
 
-        const url = this._url + '/deleteFeatures'
-
-        this._console.info('ArcGIS deleteFeatures by event url ' + url + ', ' + this._config.observationIdField + ': ' + id)
-
-        const form = new FormData()
-
-        if (this._config.eventIdField == null) {
-            form.append('where', this._config.observationIdField + ' LIKE\'%' + this._config.idSeparator + id + '\'')
-        } else {
-            form.append('where', this._config.eventIdField + '=' + id)
-        }
-
-        this.sendDelete(id);
-    }
-
-    /**
-     * Creates an add observation response handler.
-     * @param observations The observations sent.
-     * @returns The response handler.
-     */
-    private addResponseHandler(observations: ArcObjects): (chunk: any) => void {
-        return this.responseHandler(observations, false)
-    }
-
-    /**
-     * Creates an update observation response handler.
-     * @param observations The observations sent.
-     * @returns The response handler.
-     */
-    private updateResponseHandler(observations: ArcObjects): (chunk: any) => void {
-        return this.responseHandler(observations, true)
+        request(`${this._url}/deleteFeatures`, {
+            httpMethod: 'POST',
+            authentication: this._identityManager,
+            params: {
+                where: !!this._config.eventIdField 
+                    ? this._config.eventIdField + '=' + id 
+                    : this._config.observationIdField + ' LIKE\'%' + this._config.idSeparator + id + '\''
+            }
+        }).catch((error) => this._console.error('Error in ObservationSender.sendDeleteEvent :: ' + error));
     }
 
     /**
@@ -157,7 +122,7 @@ export class ObservationsSender {
      * @param update The update or add flag value.
      * @returns The response handler.
      */
-    private responseHandler(observations: ArcObjects, update: boolean): (chunk: any) => void {
+    private responseHandler(observations: ArcObjects, update?: boolean): (chunk: any) => void {
         const console = this._console
         return (chunk: any) => {
             console.log('ArcGIS ' + (update ? 'Update' : 'Add') + ' Response: ' + JSON.stringify(chunk))
@@ -207,7 +172,6 @@ export class ObservationsSender {
      */
     private queryAndUpdateAttachments(observation: ArcObservation, objectId: number) {
         // Query for existing attachments
-        const queryUrl = this._url + '/' + objectId + '/attachments'
         getAttachments({
             url: this._url,
             authentication: this._identityManager,
@@ -225,7 +189,6 @@ export class ObservationsSender {
      * @param attachmentInfos The arc attachment infos.
      */
     private updateAttachments(observation: ArcObservation, objectId: number, attachmentInfos: AttachmentInfo[]) {
-
         // Build a mapping between existing arc attachment names and the attachment infos
         let nameAttachments = new Map<string, AttachmentInfo>()
         if (attachmentInfos != null) {
@@ -319,7 +282,6 @@ export class ObservationsSender {
      * @param attachmentInfos The arc attachment infos.
      */
     private deleteAttachments(objectId: number, attachmentInfos: AttachmentInfo[]) {
-
         const attachmentIds: number[] = []
 
         for (const attachmentInfo of attachmentInfos) {