Fixes issues caused by cloning tables into Azurite from Azure Storage (…

…#2315)

* added check to remove etag from property set when writing to storage, need tests
* added test for dropping etag on write

ChangeLog.md

@@ -8,7 +8,7 @@
 
 General:
 
-- Add `--inMemoryPersistence`  and `--extentMemoryLimit` options and related configs to store all data in-memory without disk persistence. (issue #2227)
+- Add `--inMemoryPersistence` and `--extentMemoryLimit` options and related configs to store all data in-memory without disk persistence. (issue #2227)
 
 Blob:
 
@@ -17,6 +17,7 @@ Blob:
 Table:
 
 - Fixed table sas request failure with table name include upper case letter (Issue #1359)
+- Filters etag from entity writes - seen when some tools clone tables (issue #1536)
 
 ## 2023.10 Version 3.27.0
 

src/table/handlers/TableHandler.ts

@@ -192,10 +192,13 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
     // const partitionKey = this.getAndCheckPartitionKey(tableContext);
     // const rowKey = this.getAndCheckRowKey(tableContext);
     if (
+      options.tableEntityProperties == undefined ||
       !options.tableEntityProperties ||
       // rowKey and partitionKey may be empty string
       options.tableEntityProperties.PartitionKey === null ||
-      options.tableEntityProperties.RowKey === null
+      options.tableEntityProperties.PartitionKey == undefined ||
+      options.tableEntityProperties.RowKey === null ||
+      options.tableEntityProperties.RowKey === undefined
     ) {
       throw StorageErrorFactory.getPropertiesNeedValue(context);
     }
@@ -205,11 +208,18 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
     this.validateKey(context, options.tableEntityProperties.RowKey);
 
     this.checkProperties(context, options.tableEntityProperties);
+
+    // need to remove the etags from the properties to avoid errors
+    // https://docs.microsoft.com/en-us/rest/api/storageservices/insert-entity
+    options.tableEntityProperties = this.removeEtagProperty(
+      options.tableEntityProperties
+    );
+
     const entity: Entity = this.createPersistedEntity(
       context,
       options,
-      options.tableEntityProperties.PartitionKey,
-      options.tableEntityProperties.RowKey
+      options.tableEntityProperties?.PartitionKey,
+      options.tableEntityProperties?.RowKey
     );
     let normalizedEntity;
     try {
@@ -246,8 +256,8 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
         account,
         table,
         this.getOdataAnnotationUrlPrefix(tableContext, account),
-        options.tableEntityProperties.PartitionKey,
-        options.tableEntityProperties.RowKey,
+        options.tableEntityProperties?.PartitionKey,
+        options.tableEntityProperties?.RowKey,
         accept
       );
 
@@ -400,6 +410,9 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
     // check that key properties are valid
     this.validateKey(context, partitionKey);
     this.validateKey(context, rowKey);
+    options.tableEntityProperties = this.removeEtagProperty(
+      options.tableEntityProperties
+    );
 
     const entity: Entity = this.createPersistedEntity(
       context,
@@ -462,6 +475,9 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
     );
 
     this.checkMergeRequest(options, context, partitionKey, rowKey);
+    options.tableEntityProperties = this.removeEtagProperty(
+      options.tableEntityProperties
+    );
 
     const entity: Entity = this.createPersistedEntity(
       context,
@@ -1146,4 +1162,27 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
       throw StorageErrorFactory.getEntityTooLarge(context);
     }
   }
+
+  /**
+   * remove the etag property to avoid duplicate odata.etag error
+   *
+   * @private
+   * @param {{
+   *       [propertyName: string]: any;
+   *     }} tableEntityProperties
+   * @return {*}  {({ [propertyName: string]: any } | undefined)}
+   * @memberof TableHandler
+   */
+  private removeEtagProperty(
+    tableEntityProperties:
+      | {
+          [propertyName: string]: any;
+        }
+      | undefined
+  ): { [propertyName: string]: any } | undefined {
+    if (tableEntityProperties) {
+      delete tableEntityProperties["odata.etag"];
+    }
+    return tableEntityProperties;
+  }
 }

tests/table/apis/table.entity.test.ts

@@ -1,6 +1,5 @@
 import * as assert from "assert";
 import * as Azure from "azure-storage";
-
 import { configLogger } from "../../../src/common/Logger";
 import StorageError from "../../../src/table/errors/StorageError";
 
@@ -1640,7 +1639,6 @@ describe("table Entity APIs test - using Azure-Storage", () => {
                   (error, result) => {
                     if (error) {
                       assert.fail(error.message);
-                      done();
                     } else {
                       assert.strictEqual(result.PartitionKey._, partitionKey);
                       assert.strictEqual(result.RowKey._, rowKey);
@@ -1651,12 +1649,81 @@ describe("table Entity APIs test - using Azure-Storage", () => {
                 );
               } else {
                 assert.fail(mergeError.message);
-                done();
               }
             }
           );
         }
       }
     );
   });
+
+  // for github issue #1536
+  it("37. Should drop etag property when inserting entity, @loki", (done) => {
+    const dropEtagPKey = getUniqueName("drop");
+    const rowKey1 = getUniqueName("rk1");
+    const entityInsert = new TestEntity(dropEtagPKey, rowKey1, "value");
+    tableService.insertEntity(
+      tableName,
+      entityInsert,
+      (insertError, insertResult, insertResponse) => {
+        if (!insertError) {
+          tableService.retrieveEntity<TestEntity>(
+            tableName,
+            entityInsert.PartitionKey._,
+            entityInsert.RowKey._,
+            (queryError, queryResult, queryResponse) => {
+              if (!queryError) {
+                assert.strictEqual(queryResponse.statusCode, 200);
+                assert.strictEqual(
+                  queryResult.myValue._,
+                  entityInsert.myValue._
+                );
+                // now add odata etag property to the entity
+                const entityWithEtag = queryResult;
+                const rowKey2 = getUniqueName("rk2");
+                entityWithEtag.RowKey._ = rowKey2;
+                (entityWithEtag as any)["odata.etag"] =
+                  "W/\"datetime'2021-06-30T00%3A00%3A00.0000000Z'\"";
+                tableService.insertEntity(
+                  tableName,
+                  entityWithEtag,
+                  (insert2Error, insert2Result, insert2Response) => {
+                    if (!insert2Error) {
+                      assert.strictEqual(insert2Response.statusCode, 201);
+                      tableService.retrieveEntity<TestEntity>(
+                        tableName,
+                        entityWithEtag.PartitionKey._,
+                        entityWithEtag.RowKey._,
+                        (query2Error, query2Result, query2Response) => {
+                          if (!query2Error && query2Result && query2Response) {
+                            assert.strictEqual(query2Response.statusCode, 200);
+                            assert.strictEqual(
+                              query2Result.myValue._,
+                              entityInsert.myValue._
+                            );
+                            assert.notDeepStrictEqual(
+                              (query2Response as any).body["odata.etag"],
+                              "W/\"datetime'2021-06-30T00%3A00%3A00.0000000Z'\"",
+                              "Etag value is not writable and should be dropped."
+                            );
+                            done();
+                          } else {
+                            assert.fail(query2Error.message);
+                          }
+                        }
+                      );
+                    }
+                  }
+                );
+              } else {
+                assert.fail(queryError.message);
+              }
+            }
+          );
+        } else {
+          assert.fail(insertError.message);
+        }
+      }
+    );
+  });
 });