From 21a8e1f46d477c7c81bc94be4be6eb61c730849a Mon Sep 17 00:00:00 2001 From: Dennis van der Wals Date: Thu, 5 Oct 2023 19:05:08 +0200 Subject: [PATCH 1/4] Added flag for single item generation with array values --- .../jsonoutputenhanced/JsonOutput.java | 253 ++++++++++++------ .../jsonoutputenhanced/JsonOutputDialog.java | 32 ++- .../jsonoutputenhanced/JsonOutputField.java | 2 + .../jsonoutputenhanced/JsonOutputMeta.java | 20 ++ .../messages/messages_en_US.properties | 5 +- 5 files changed, 222 insertions(+), 90 deletions(-) diff --git a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutput.java b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutput.java index bbbdeb77b89..5faf5474d28 100644 --- a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutput.java +++ b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutput.java @@ -17,10 +17,13 @@ package org.apache.hop.pipeline.transforms.jsonoutputenhanced; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.vfs2.FileObject; import org.apache.hop.core.Const; import org.apache.hop.core.ResultFile; @@ -40,14 +43,11 @@ import org.apache.hop.pipeline.PipelineMeta; import org.apache.hop.pipeline.transform.BaseTransform; import org.apache.hop.pipeline.transform.TransformMeta; - -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.List; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; public class JsonOutput extends BaseTransform { private static final Class PKG = @@ -56,6 +56,7 @@ public class JsonOutput extends BaseTransform { public Object[] prevRow; private JsonNodeFactory nc; private ObjectMapper mapper; + private ObjectNode currentNode; public JsonOutput( TransformMeta transformMeta, @@ -107,15 +108,14 @@ public boolean processRow() throws HopException { // Let's output the remaining unsafe data outputRow(prevRow); // only attempt writing to file when the first row is not empty - if (data.isWriteToFile && !first) writeJsonFile(); + if (data.isWriteToFile && !first) { + writeJsonFile(); + } setOutputDone(); return false; } - if (first) { - - if (onFirstRecord(r)) return false; - } + if (first && onFirstRecord(r)) return false; data.rowsAreSafe = false; @@ -127,83 +127,151 @@ public boolean processRow() throws HopException { public void manageRowItems(Object[] row) throws HopException { ObjectNode itemNode; + + boolean sameGroup = sameGroup(prevRow, row); + + if (meta.isUseSingleItemPerGroup()) { + /* + * If grouped rows are forced to produce a single item, reuse the same itemNode as long as the + * row belongs to the previous group. Feature #3287 + */ + if (!sameGroup || currentNode == null) { + currentNode = new ObjectNode(nc); + } + + itemNode = currentNode; + + } else { + // Create a new object with specified fields + itemNode = new ObjectNode(nc); + } - if (!sameGroup(prevRow, row) && data.jsonKeyGroupItems.size() > 0) { + if (!sameGroup && data.jsonKeyGroupItems.size() > 0) { // Output the new row logDebug("Record Num: " + data.nrRow + " - Generating JSON chunk"); outputRow(prevRow); data.jsonKeyGroupItems = new ArrayList<>(); } - // Create a new object with specified fields - itemNode = new ObjectNode(nc); - for (int i = 0; i < data.nrFields; i++) { JsonOutputField outputField = meta.getOutputFields()[i]; - IValueMeta v = data.inputRowMeta.getValueMeta(data.fieldIndexes[i]); + String jsonAttributeName = getJsonAttributeName(outputField); + boolean putBlank = !outputField.isRemoveIfBlank(); + + /* + * Prepare the array node to collect all values of a field inside a group into an array. Skip + * fields appearing in the grouped fields since they are always unique per group. + */ + ArrayNode arNode = null; + if (meta.isUseSingleItemPerGroup() && !outputField.isKeyField) { + if (!itemNode.has(jsonAttributeName)) { + arNode = itemNode.putArray(jsonAttributeName); + } else { + arNode = (ArrayNode) itemNode.get(jsonAttributeName); + } + // In case whe have an array to store data, the flag to remove blanks is effectivly deactivated. + putBlank = false; + } + IValueMeta v = data.inputRowMeta.getValueMeta(data.fieldIndexes[i]); switch (v.getType()) { case IValueMeta.TYPE_BOOLEAN: Boolean boolValue = data.inputRowMeta.getBoolean(row, data.fieldIndexes[i]); - if (boolValue != null) itemNode.put(getJsonAttributeName(outputField), boolValue); - else { - if (!outputField.isRemoveIfBlank()) - itemNode.put(getJsonAttributeName(outputField), boolValue); + if (putBlank) { + itemNode.put(jsonAttributeName, boolValue); + } else if (boolValue != null) { + if (arNode == null) { + itemNode.put(jsonAttributeName, boolValue); + } else { + arNode.add(boolValue); + } } break; case IValueMeta.TYPE_INTEGER: Long integerValue = data.inputRowMeta.getInteger(row, data.fieldIndexes[i]); - if (integerValue != null) itemNode.put(getJsonAttributeName(outputField), integerValue); - else if (!outputField.isRemoveIfBlank()) - itemNode.put(getJsonAttributeName(outputField), integerValue); + if (putBlank) { + itemNode.put(jsonAttributeName, integerValue); + } else if (integerValue != null) { + if (arNode == null) { + itemNode.put(jsonAttributeName, integerValue); + } else { + arNode.add(integerValue); + } + } break; case IValueMeta.TYPE_NUMBER: Double numberValue = data.inputRowMeta.getNumber(row, data.fieldIndexes[i]); - if (numberValue != null) itemNode.put(getJsonAttributeName(outputField), numberValue); - else if (!outputField.isRemoveIfBlank()) - itemNode.put(getJsonAttributeName(outputField), numberValue); + if (putBlank) { + itemNode.put(jsonAttributeName, numberValue); + } else if (numberValue != null) { + if (arNode == null) { + itemNode.put(jsonAttributeName, numberValue); + } else { + arNode.add(numberValue); + } + } break; case IValueMeta.TYPE_BIGNUMBER: BigDecimal bignumberValue = data.inputRowMeta.getBigNumber(row, data.fieldIndexes[i]); - if (bignumberValue != null) - itemNode.put(getJsonAttributeName(outputField), bignumberValue); - else if (!outputField.isRemoveIfBlank()) - itemNode.put(getJsonAttributeName(outputField), bignumberValue); + if (putBlank) { + itemNode.put(jsonAttributeName, bignumberValue); + } else if (bignumberValue != null) { + if (arNode == null) { + itemNode.put(jsonAttributeName, bignumberValue); + } else { + arNode.add(bignumberValue); + } + } break; default: String value = data.inputRowMeta.getString(row, data.fieldIndexes[i]); - if (value != null) { + if (putBlank) { + itemNode.put(jsonAttributeName, value); + } else if (value != null) { if (outputField.isJSONFragment()) { try { JsonNode jsonNode = mapper.readTree(value); if (outputField.isWithoutEnclosing()) { itemNode.setAll((ObjectNode) jsonNode); } else { - itemNode.put(getJsonAttributeName(outputField), jsonNode); + if (arNode == null) { + itemNode.set(jsonAttributeName, jsonNode); + } else { + arNode.add(jsonNode); + } } } catch (IOException e) { throw new HopTransformException( BaseMessages.getString(PKG, "JsonOutput.Error.Casting"), e); } } else { - itemNode.put(getJsonAttributeName(outputField), value); + if (arNode == null) { + itemNode.put(jsonAttributeName, value); + } else { + arNode.add(value); + } } - } else { - if (!outputField.isRemoveIfBlank()) - itemNode.put(getJsonAttributeName(outputField), value); } break; } } - data.jsonKeyGroupItems.add(itemNode); + /* + * Only add a new item node if each row should produce a single JSON object or in case of a + * single JSON object for a group of rows, if no item node was added yet. This happens for the + * first new row of a group only. + */ + if (!meta.isUseSingleItemPerGroup() || data.jsonKeyGroupItems.size() == 0) { + data.jsonKeyGroupItems.add(itemNode); + } + prevRow = data.inputRowMeta.cloneRow(row); // copy the row to previous data.nrRow++; @@ -226,7 +294,6 @@ private String getKeyJsonAttributeName(JsonOutputKeyField field) { return (elementName != null && elementName.length() > 0 ? elementName : field.getFieldName()); } - @SuppressWarnings("unchecked") private void outputRow(Object[] rowData) throws HopException { // We can now output an object ObjectNode globalItemNode = null; @@ -244,7 +311,9 @@ private void outputRow(Object[] rowData) throws HopException { Object[] keyRow = new Object[meta.getKeyFields().length]; // Create a new object with specified fields - if (data.isWriteToFile) globalItemNode = new ObjectNode(nc); + if (data.isWriteToFile) { + globalItemNode = new ObjectNode(nc); + } for (int i = 0; i < meta.getKeyFields().length; i++) { try { @@ -252,33 +321,38 @@ private void outputRow(Object[] rowData) throws HopException { switch (vmi.getType()) { case IValueMeta.TYPE_BOOLEAN: keyRow[i] = data.inputRowMeta.getBoolean(rowData, data.keysGroupIndexes[i]); - if (data.isWriteToFile) + if (data.isWriteToFile) { globalItemNode.put( getKeyJsonAttributeName(meta.getKeyFields()[i]), (Boolean) keyRow[i]); + } break; case IValueMeta.TYPE_INTEGER: keyRow[i] = data.inputRowMeta.getInteger(rowData, data.keysGroupIndexes[i]); - if (data.isWriteToFile) + if (data.isWriteToFile) { globalItemNode.put( getKeyJsonAttributeName(meta.getKeyFields()[i]), (Long) keyRow[i]); + } break; case IValueMeta.TYPE_NUMBER: keyRow[i] = data.inputRowMeta.getNumber(rowData, data.keysGroupIndexes[i]); - if (data.isWriteToFile) + if (data.isWriteToFile) { globalItemNode.put( getKeyJsonAttributeName(meta.getKeyFields()[i]), (Double) keyRow[i]); + } break; case IValueMeta.TYPE_BIGNUMBER: keyRow[i] = data.inputRowMeta.getBigNumber(rowData, data.keysGroupIndexes[i]); - if (data.isWriteToFile) + if (data.isWriteToFile) { globalItemNode.put( getKeyJsonAttributeName(meta.getKeyFields()[i]), (BigDecimal) keyRow[i]); + } break; default: keyRow[i] = data.inputRowMeta.getString(rowData, data.keysGroupIndexes[i]); - if (data.isWriteToFile) + if (data.isWriteToFile) { globalItemNode.put( getKeyJsonAttributeName(meta.getKeyFields()[i]), (String) keyRow[i]); + } break; } } catch (HopValueException e) { @@ -291,7 +365,9 @@ private void outputRow(Object[] rowData) throws HopException { // TODO: Maybe there will be an opportunity for better code here without going through // JSON serialization here... JsonNode jsonNode = mapper.readTree(data.jsonSerialized); - if (meta.getOutputValue() != null) globalItemNode.put(meta.getOutputValue(), jsonNode); + if (meta.getOutputValue() != null) { + globalItemNode.set(meta.getOutputValue(), jsonNode); + } } catch (IOException e) { // TBD Exception must be properly managed e.printStackTrace(); @@ -322,10 +398,9 @@ private void outputRow(Object[] rowData) throws HopException { private void writeJsonFile() throws HopTransformException { // Open a file - if (data.isWriteToFile && !openNewFile()) { + if (data.isWriteToFile && !openNewFile()) throw new HopTransformException( BaseMessages.getString(PKG, "JsonOutput.Error.OpenNewFile", buildFilename())); - } // Write data to file try { data.writer.write(data.jsonSerialized); @@ -343,37 +418,32 @@ private void serializeJson(List jsonItemsList) { try { if (meta.getJsonBloc() != null && meta.getJsonBloc().length() > 0) { // TBD Try to understand if this can have a performance impact and do it better... - theNode.put( - meta.getJsonBloc(), - mapper.readTree( - mapper.writeValueAsString( - jsonItemsList.size() > 1 + theNode.set(meta.getJsonBloc(), + mapper.readTree(mapper.writeValueAsString(jsonItemsList.size() > 1 ? jsonItemsList + : (!meta.isUseArrayWithSingleInstance() ? jsonItemsList.get(0) : jsonItemsList)))); + if (meta.isJsonPrittified()) { + data.jsonSerialized = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(theNode); + } else { + data.jsonSerialized = mapper.writeValueAsString(theNode); + } + } else if (meta.isJsonPrittified()) { + data.jsonSerialized = + mapper + .writerWithDefaultPrettyPrinter() + .writeValueAsString( + (jsonItemsList.size() > 1 ? jsonItemsList : (!meta.isUseArrayWithSingleInstance() ? jsonItemsList.get(0) - : jsonItemsList)))); - if (meta.isJsonPrittified()) - data.jsonSerialized = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(theNode); - else data.jsonSerialized = mapper.writeValueAsString(theNode); + : jsonItemsList))); } else { - if (meta.isJsonPrittified()) - data.jsonSerialized = - mapper - .writerWithDefaultPrettyPrinter() - .writeValueAsString( - (jsonItemsList.size() > 1 - ? jsonItemsList - : (!meta.isUseArrayWithSingleInstance() - ? jsonItemsList.get(0) - : jsonItemsList))); - else - data.jsonSerialized = - mapper.writeValueAsString( - (jsonItemsList.size() > 1 - ? jsonItemsList - : (!meta.isUseArrayWithSingleInstance() - ? jsonItemsList.get(0) - : jsonItemsList))); + data.jsonSerialized = + mapper.writeValueAsString( + (jsonItemsList.size() > 1 + ? jsonItemsList + : (!meta.isUseArrayWithSingleInstance() + ? jsonItemsList.get(0) + : jsonItemsList))); } } catch (IOException e) { // TBD Exception must be properly managed @@ -433,11 +503,21 @@ private void initDataFieldsPositionsArray() throws HopException { for (int i = 0; i < data.nrFields; i++) { data.fieldIndexes[i] = data.inputRowMeta.indexOfValue(meta.getOutputFields()[i].getFieldName()); - if (data.fieldIndexes[i] < 0) { + if (data.fieldIndexes[i] < 0) throw new HopException(BaseMessages.getString(PKG, "JsonOutput.Exception.FieldNotFound")); - } JsonOutputField field = meta.getOutputFields()[i]; field.setElementName(variables.resolve(field.getElementName())); + + /* + * Mark all output fields that are part of the group key fields. This way we can avoid + * collecting unique values of each group inside an array. Feature #3287 + */ + for (JsonOutputKeyField jsonOutputKeyField : meta.getKeyFields()) { + if (jsonOutputKeyField.getFieldName().equals(field.getFieldName())) { + field.isKeyField = true; + break; + } + } } } @@ -468,9 +548,8 @@ public void dispose() { } private void createParentFolder(String filename) throws HopTransformException { - if (!meta.isCreateParentFolder()) { + if (!meta.isCreateParentFolder()) return; - } // Check for parent folder FileObject parentfolder = null; try { @@ -504,9 +583,8 @@ private void createParentFolder(String filename) throws HopTransformException { public boolean openNewFile() { - if (data.writer != null) { + if (data.writer != null) return true; - } boolean retval = false; try { @@ -556,9 +634,8 @@ public String buildFilename() { } private boolean closeFile() { - if (data.writer == null) { + if (data.writer == null) return true; - } boolean retval = false; try { diff --git a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutputDialog.java b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutputDialog.java index 6db3b5456b6..61434125db8 100644 --- a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutputDialog.java +++ b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutputDialog.java @@ -80,6 +80,7 @@ public class JsonOutputDialog extends BaseTransformDialog implements ITransformD private TextVar wOutputValue; private Button wUseArrayWithSingleInstance; + private Button wUseSingleItemPerGroup; private Label wlBlocName; private TextVar wBlocName; @@ -569,13 +570,40 @@ public void widgetSelected(SelectionEvent e) { input.setChanged(); } }); + + Label wlUseSingleItemPerGroup = new Label(wSettings, SWT.RIGHT); + wlUseSingleItemPerGroup.setText( + BaseMessages.getString(PKG, "JsonOutputDialog.UseSingleItemPerGroup.Label")); + PropsUi.setLook(wlUseSingleItemPerGroup); + FormData fdlUseSingleItemPerGroup = new FormData(); + fdlUseSingleItemPerGroup.left = new FormAttachment(0, 0); + fdlUseSingleItemPerGroup.top = new FormAttachment(wUseArrayWithSingleInstance, margin); + fdlUseSingleItemPerGroup.right = new FormAttachment(middle, -margin); + wlUseSingleItemPerGroup.setLayoutData(fdlUseSingleItemPerGroup); + wUseSingleItemPerGroup = new Button(wSettings, SWT.CHECK); + wUseSingleItemPerGroup.setToolTipText( + BaseMessages.getString(PKG, "JsonOutputDialog.UseSingleItemPerGroup.Tooltip")); + PropsUi.setLook(wUseSingleItemPerGroup); + FormData fdUseSingleItemPerGroup = new FormData(); + fdUseSingleItemPerGroup.left = new FormAttachment(middle, 0); + fdUseSingleItemPerGroup.top = + new FormAttachment(wlUseSingleItemPerGroup, 0, SWT.CENTER); + fdUseSingleItemPerGroup.right = new FormAttachment(100, 0); + wUseSingleItemPerGroup.setLayoutData(fdUseSingleItemPerGroup); + wUseSingleItemPerGroup.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + input.setChanged(); + } + }); Label wlJSONPrittified = new Label(wSettings, SWT.RIGHT); wlJSONPrittified.setText(BaseMessages.getString(PKG, "JsonOutputDialog.JSONPrittified.Label")); PropsUi.setLook(wlJSONPrittified); FormData fdlJSONPrittified = new FormData(); fdlJSONPrittified.left = new FormAttachment(0, 0); - fdlJSONPrittified.top = new FormAttachment(wUseArrayWithSingleInstance, margin); + fdlJSONPrittified.top = new FormAttachment(wUseSingleItemPerGroup, margin); fdlJSONPrittified.right = new FormAttachment(middle, -margin); wlJSONPrittified.setLayoutData(fdlJSONPrittified); wJSONPrittified = new Button(wSettings, SWT.CHECK); @@ -984,6 +1012,7 @@ private void getData() { wEncoding.setText(Const.NVL(input.getEncoding(), "")); wOutputValue.setText(Const.NVL(input.getOutputValue(), "")); wUseArrayWithSingleInstance.setSelection(input.isUseArrayWithSingleInstance()); + wUseSingleItemPerGroup.setSelection(input.isUseSingleItemPerGroup()); wJSONPrittified.setSelection(input.isJsonPrittified()); wSplitOutputAfter.setText(Integer.toString(input.getSplitOutputAfter())); wOperation.setText(JsonOutputMeta.getOperationTypeDesc(input.getOperationType())); @@ -1061,6 +1090,7 @@ private void getInfo(JsonOutputMeta jsometa) { jsometa.setEncoding(wEncoding.getText()); jsometa.setOutputValue(wOutputValue.getText()); jsometa.setUseArrayWithSingleInstance(wUseArrayWithSingleInstance.getSelection()); + jsometa.setUseSingleItemPerGroup(wUseSingleItemPerGroup.getSelection()); jsometa.setOperationType(JsonOutputMeta.getOperationTypeByDesc(wOperation.getText())); jsometa.setJsonPrittified(wJSONPrittified.getSelection()); jsometa.setSplitOutputAfter( diff --git a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutputField.java b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutputField.java index f9fb00ccdb7..17c36987e1f 100644 --- a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutputField.java +++ b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutputField.java @@ -35,6 +35,8 @@ public class JsonOutputField implements Cloneable { @Injection(name = "JSON_REMOVEIFBLANK", group = "FIELDS") private boolean removeIfBlank; + + public boolean isKeyField; public boolean isJSONFragment() { return isJSONFragment; diff --git a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutputMeta.java b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutputMeta.java index 6244ce64540..0a8768d76f6 100644 --- a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutputMeta.java +++ b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutputMeta.java @@ -108,6 +108,10 @@ public class JsonOutputMeta extends BaseFileOutputMeta Date: Fri, 13 Oct 2023 13:55:11 +0200 Subject: [PATCH 2/4] update documentation --- .../ROOT/pages/pipeline/transforms/enhancedjsonoutput.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/enhancedjsonoutput.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/enhancedjsonoutput.adoc index 4bf18019b42..b5ebf192c84 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/enhancedjsonoutput.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/enhancedjsonoutput.adoc @@ -65,6 +65,7 @@ Can be empty string that will affect output json structure, see detailed descrip |Output value|This value will be used as a transform output field. Will contain generated json output block depending on transform settings. |Force Arrays In JSON| If checked, JSON output will be forced as a single item whose content is an array. +|Force single grouped Item|Grouped values are stored in an array inside the JSON instead of creating an array of JSONs |Pritty Print JSON|If checked, JSON output will be pritty printed. |=== From dd3cd39b8daf099ed0eba764a1c4d0e42e8a3bfc Mon Sep 17 00:00:00 2001 From: Dennis van der Wals Date: Thu, 9 Nov 2023 16:02:36 +0100 Subject: [PATCH 3/4] Added integration tests for validating current and new behaviour --- ...nced-json-output-grouping-unit-testing.hpl | 436 ++++++++++++ .../0069-enhanced-json-output-grouping.hpl | 648 ++++++++++++++++++ .../datasets/golden-json-grouped-output.csv | 4 + ...golden-json-grouped-single-item-output.csv | 4 + .../datasets/input-enhanced-json-test.csv | 6 + .../main-0069-enhanced-json-grouping.hwf | 147 ++++ .../dataset/golden-json-grouped-output.json | 24 + ...olden-json-grouped-single-item-output.json | 24 + .../dataset/input-enhanced-json-test.csv.json | 24 + .../0069-enhanced-json-grouping-test.json | 70 ++ 10 files changed, 1387 insertions(+) create mode 100644 integration-tests/transforms/0069-enhanced-json-output-grouping-unit-testing.hpl create mode 100644 integration-tests/transforms/0069-enhanced-json-output-grouping.hpl create mode 100644 integration-tests/transforms/datasets/golden-json-grouped-output.csv create mode 100644 integration-tests/transforms/datasets/golden-json-grouped-single-item-output.csv create mode 100644 integration-tests/transforms/datasets/input-enhanced-json-test.csv create mode 100644 integration-tests/transforms/main-0069-enhanced-json-grouping.hwf create mode 100644 integration-tests/transforms/metadata/dataset/golden-json-grouped-output.json create mode 100644 integration-tests/transforms/metadata/dataset/golden-json-grouped-single-item-output.json create mode 100644 integration-tests/transforms/metadata/dataset/input-enhanced-json-test.csv.json create mode 100644 integration-tests/transforms/metadata/unit-test/0069-enhanced-json-grouping-test.json diff --git a/integration-tests/transforms/0069-enhanced-json-output-grouping-unit-testing.hpl b/integration-tests/transforms/0069-enhanced-json-output-grouping-unit-testing.hpl new file mode 100644 index 00000000000..62dfba91b6c --- /dev/null +++ b/integration-tests/transforms/0069-enhanced-json-output-grouping-unit-testing.hpl @@ -0,0 +1,436 @@ + + + + + 0069-enhanced-json-output-grouping-unit-testing + Y + + + + Normal + + + N + 1000 + 100 + - + 2023/11/09 15:12:39.187 + - + 2023/11/09 15:12:39.187 + + + + + + Sort rows + json array of multiple results + Y + + + Sort rows + single json result with value array + Y + + + CSV file input + Sort rows + Y + + + json array of multiple results + Replace comma + Y + + + single json result with value array + Replace comma 2 + Y + + + Replace comma + Select values + Y + + + Select values + verify_grouping + Y + + + Replace comma 2 + Select values 2 + Y + + + Select values 2 + verify_single_item_grouping + Y + + + + CSV file input + CSVInput + + Y + + 1 + + none + + + + + + N + , + " +
Y
+ 50000 + Y + N + N + N + + + + group + String + + + + + -1 + -1 + none + + + value + Integer + + + + + -1 + -1 + none + + + + + 208 + 192 + + + + Replace comma + ReplaceString + + Y + + 1 + + none + + + + + N + result + N + result + ; + , + N + N + N + + + + + 800 + 128 + + + + Replace comma 2 + ReplaceString + + Y + + 1 + + none + + + + + N + result + N + result + ; + , + N + N + N + + + + + 800 + 256 + + + + Select values + SelectValues + + Y + + 1 + + none + + + + + group + + + result_1 + result + + N + + + + 976 + 128 + + + + Select values 2 + SelectValues + + Y + + 1 + + none + + + + + group + + + result_1 + result + + N + + + + 976 + 256 + + + + Sort rows + SortRows + + N + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + N + + + group + Y + N + N + 0 + N + + + + + 384 + 192 + + + + json array of multiple results + EnhancedJsonOutput + + Y + + 1 + + none + + + result + result + outputvalue + N + N + N + UTF-8 + N + + + 0 + json + N + N + N + N + N + N + N + + + + + + + group + + + + + + value + value + N + N + N + + + + + 592 + 128 + + + + single json result with value array + EnhancedJsonOutput + + N + + 1 + + none + + + result + result + outputvalue + N + Y + N + UTF-8 + N + + + 0 + json + N + N + N + N + N + N + N + + + + + + + group + + + + + + value + value + N + N + N + + + + + 592 + 256 + + + + verify_grouping + Dummy + + Y + + 1 + + none + + + + + 1184 + 128 + + + + verify_single_item_grouping + Dummy + + Y + + 1 + + none + + + + + 1184 + 256 + + + + + + diff --git a/integration-tests/transforms/0069-enhanced-json-output-grouping.hpl b/integration-tests/transforms/0069-enhanced-json-output-grouping.hpl new file mode 100644 index 00000000000..9e4fbe541ce --- /dev/null +++ b/integration-tests/transforms/0069-enhanced-json-output-grouping.hpl @@ -0,0 +1,648 @@ + + + + + 0069-enhanced-json-output-grouping + Y + + + + Normal + + + N + 1000 + 100 + - + 2023/11/07 20:56:22.463 + - + 2023/11/07 20:56:22.463 + + + + + + Generate rows + Clone row + Y + + + Clone row + Fake data + Y + + + Fake data + Sort rows + Y + + + Sort rows + json array of multiple results + Y + + + Sort rows + single json result with value array + Y + + + json array of multiple results + parse json array + Y + + + single json result with value array + parse single json object + Y + + + parse single json object + count single objects + Y + + + parse json array + count objects + Y + + + count objects + expect 100 rows + Y + + + count single objects + expect 10 rows + Y + + + expect 100 rows + failed multi json generation + Y + + + expect 10 rows + failed single json generation + Y + + + + Clone row + CloneRow + + Y + + 1 + + none + + + N + Y + group_key + key + N + 9 + + + 368 + 176 + + + + Fake data + Fake + + Y + + 1 + + none + + + + + random_data + username + Name + + + en + + + 512 + 176 + + + + Generate rows + RowGenerator + + Y + + 1 + + none + + + + + -1 + trigger_value + -1 + Y + String + + + 5000 + FiveSecondsAgo + 10 + N + now + + + 240 + 176 + + + + Sort rows + SortRows + + N + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + N + + + group_key + Y + N + N + 0 + N + + + + + 640 + 176 + + + + count objects + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + amount + random_data + COUNT_ALL + + + N + + + N + grp + + + 1296 + 112 + + + + count single objects + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + amount + random_data + COUNT_ALL + + + N + + + N + grp + + + 1296 + 240 + + + + expect 10 rows + FilterRows + + Y + + 1 + + none + + + + + + + = + amount + N + - + + N + -1 + ####0;-####0 + constant + 0 + 10 + Integer + + + + failed single json generation + + + 1456 + 240 + + + + expect 100 rows + FilterRows + + Y + + 1 + + none + + + + + + + = + amount + N + - + + N + -1 + ####0;-####0 + constant + 0 + 100 + Integer + + + + failed multi json generation + + + 1456 + 112 + + + + failed multi json generation + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1664 + 112 + + + + failed single json generation + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1664 + 240 + + + + json array of multiple results + EnhancedJsonOutput + + Y + + 1 + + none + + + not_grouped_jsons + result + outputvalue + N + N + N + UTF-8 + N + + + 0 + json + N + N + N + N + N + N + N + + + + + + + group_key + group_key + + + + + random_data + random_data + N + N + N + + + + + 848 + 112 + + + + parse json array + JsonInput + + Y + + 1 + + none + + + N + + N + N + N + N + N + Y + Y + Y + + + + + + N + N + + + + random_data + $.result.*.random_data + String + + + + + -1 + -1 + none + N + + + 0 + Y + N + not_grouped_jsons + + + + + + + + + + + 1072 + 112 + + + + parse single json object + JsonInput + + Y + + 1 + + none + + + N + + N + N + N + N + N + Y + Y + Y + + + + + + N + N + + + + result + $.result + String + + + + + -1 + -1 + none + N + + + random_data + $.result.random_data + JSON + + + + + -1 + -1 + none + N + + + 0 + Y + N + forced_into_single_grouped_json + + + + + + + + + + + 1088 + 240 + + + + single json result with value array + EnhancedJsonOutput + + N + + 1 + + none + + + forced_into_single_grouped_json + result + outputvalue + N + Y + N + UTF-8 + N + + + 0 + json + N + N + N + N + N + N + N + + + + + + + group_key + group_key + + + + + random_data + random_data + N + N + N + + + + + 848 + 240 + + + + + + diff --git a/integration-tests/transforms/datasets/golden-json-grouped-output.csv b/integration-tests/transforms/datasets/golden-json-grouped-output.csv new file mode 100644 index 00000000000..3a31dd5620e --- /dev/null +++ b/integration-tests/transforms/datasets/golden-json-grouped-output.csv @@ -0,0 +1,4 @@ +group,result +A,{"result":[{"value":1};{"value":2}]} +B,{"result":[{"value":3};{"value":4}]} +C,{"result":{"value":5}} \ No newline at end of file diff --git a/integration-tests/transforms/datasets/golden-json-grouped-single-item-output.csv b/integration-tests/transforms/datasets/golden-json-grouped-single-item-output.csv new file mode 100644 index 00000000000..0c750fffdaf --- /dev/null +++ b/integration-tests/transforms/datasets/golden-json-grouped-single-item-output.csv @@ -0,0 +1,4 @@ +group, result +A,{"result":{"value":[1;2]}} +B,{"result":{"value":[3;4]}} +C,{"result":{"value":[5]}} \ No newline at end of file diff --git a/integration-tests/transforms/datasets/input-enhanced-json-test.csv b/integration-tests/transforms/datasets/input-enhanced-json-test.csv new file mode 100644 index 00000000000..15d1fdea826 --- /dev/null +++ b/integration-tests/transforms/datasets/input-enhanced-json-test.csv @@ -0,0 +1,6 @@ +group,value +A,1 +A,2 +B,3 +B,4 +C,5 diff --git a/integration-tests/transforms/main-0069-enhanced-json-grouping.hwf b/integration-tests/transforms/main-0069-enhanced-json-grouping.hwf new file mode 100644 index 00000000000..50b44cbad2a --- /dev/null +++ b/integration-tests/transforms/main-0069-enhanced-json-grouping.hwf @@ -0,0 +1,147 @@ + + + + main-0069-enhanced-json-grouping + Y + + + + - + 2023/11/07 21:34:50.471 + - + 2023/11/07 21:34:50.471 + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 50 + 50 + + + + 0069-enhanced-json-output-grouping.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/0069-enhanced-json-output-grouping.hpl + Basic + + Y + + N + direct + N + N + Y + N + 304 + 48 + + + + Success + + SUCCESS + + N + 752 + 48 + + + + Failed + + ABORT + + N + N + 304 + 192 + + + + Run Pipeline Unit Tests + + RunPipelineTests + + + + 0069-enhanced-json-grouping-test + + + N + 560 + 48 + + + + + + Start + 0069-enhanced-json-output-grouping.hpl + Y + Y + Y + + + 0069-enhanced-json-output-grouping.hpl + Failed + Y + N + N + + + 0069-enhanced-json-output-grouping.hpl + Run Pipeline Unit Tests + Y + Y + N + + + Run Pipeline Unit Tests + Success + Y + Y + N + + + + + + diff --git a/integration-tests/transforms/metadata/dataset/golden-json-grouped-output.json b/integration-tests/transforms/metadata/dataset/golden-json-grouped-output.json new file mode 100644 index 00000000000..b8240140ba2 --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-json-grouped-output.json @@ -0,0 +1,24 @@ +{ + "base_filename": "golden-json-grouped-output.csv", + "name": "golden-json-grouped-output", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "group", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "result", + "field_format": "" + } + ], + "folder_name": "" +} \ No newline at end of file diff --git a/integration-tests/transforms/metadata/dataset/golden-json-grouped-single-item-output.json b/integration-tests/transforms/metadata/dataset/golden-json-grouped-single-item-output.json new file mode 100644 index 00000000000..4384f895976 --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-json-grouped-single-item-output.json @@ -0,0 +1,24 @@ +{ + "base_filename": "golden-json-grouped-single-item-output.csv", + "name": "golden-json-grouped-single-item-output", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "group", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "result", + "field_format": "" + } + ], + "folder_name": "" +} \ No newline at end of file diff --git a/integration-tests/transforms/metadata/dataset/input-enhanced-json-test.csv.json b/integration-tests/transforms/metadata/dataset/input-enhanced-json-test.csv.json new file mode 100644 index 00000000000..5b3753db93b --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/input-enhanced-json-test.csv.json @@ -0,0 +1,24 @@ +{ + "base_filename": "input-enhanced-json-test.csv", + "name": "input-enhanced-json-test.csv", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "group", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 5, + "field_precision": -1, + "field_name": "value", + "field_format": "" + } + ], + "folder_name": "" +} \ No newline at end of file diff --git a/integration-tests/transforms/metadata/unit-test/0069-enhanced-json-grouping-test.json b/integration-tests/transforms/metadata/unit-test/0069-enhanced-json-grouping-test.json new file mode 100644 index 00000000000..ec40af2a336 --- /dev/null +++ b/integration-tests/transforms/metadata/unit-test/0069-enhanced-json-grouping-test.json @@ -0,0 +1,70 @@ +{ + "variableValues": [], + "database_replacements": [], + "autoOpening": true, + "basePath": "", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "group", + "data_set_field": "group" + }, + { + "transform_field": "result", + "data_set_field": "result" + } + ], + "field_order": [ + "group", + "result" + ], + "data_set_name": "golden-json-grouped-single-item-output", + "transform_name": "verify_single_item_grouping" + }, + { + "field_mappings": [ + { + "transform_field": "group", + "data_set_field": "group" + }, + { + "transform_field": "result", + "data_set_field": "result" + } + ], + "field_order": [ + "group", + "result" + ], + "data_set_name": "golden-json-grouped-output", + "transform_name": "verify_grouping" + } + ], + "input_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "group", + "data_set_field": "group" + }, + { + "transform_field": "value", + "data_set_field": "value" + } + ], + "field_order": [ + "group", + "value" + ], + "data_set_name": "input-enhanced-json-test.csv", + "transform_name": "CSV file input" + } + ], + "name": "0069-enhanced-json-grouping-test", + "description": "", + "persist_filename": "", + "trans_test_tweaks": [], + "pipeline_filename": "./0069-enhanced-json-output-grouping-unit-testing.hpl", + "test_type": "UNIT_TEST" +} \ No newline at end of file From 7b8cbb17ca041e7b818d6c073e95e25261febc7b Mon Sep 17 00:00:00 2001 From: Dennis van der Wals Date: Thu, 9 Nov 2023 16:08:48 +0100 Subject: [PATCH 4/4] Merged new tests into existing transformation test --- ...ced-json-output-grouping-unit-testing.hpl} | 0 ...=> 0068-enhanced-json-output-grouping.hpl} | 0 .../main-0068-enhanced-json-2927.hwf | 90 ++++++++++- .../main-0069-enhanced-json-grouping.hwf | 147 ------------------ ... => 0068-enhanced-json-grouping-test.json} | 4 +- 5 files changed, 85 insertions(+), 156 deletions(-) rename integration-tests/transforms/{0069-enhanced-json-output-grouping-unit-testing.hpl => 0068-enhanced-json-output-grouping-unit-testing.hpl} (100%) rename integration-tests/transforms/{0069-enhanced-json-output-grouping.hpl => 0068-enhanced-json-output-grouping.hpl} (100%) delete mode 100644 integration-tests/transforms/main-0069-enhanced-json-grouping.hwf rename integration-tests/transforms/metadata/unit-test/{0069-enhanced-json-grouping-test.json => 0068-enhanced-json-grouping-test.json} (93%) diff --git a/integration-tests/transforms/0069-enhanced-json-output-grouping-unit-testing.hpl b/integration-tests/transforms/0068-enhanced-json-output-grouping-unit-testing.hpl similarity index 100% rename from integration-tests/transforms/0069-enhanced-json-output-grouping-unit-testing.hpl rename to integration-tests/transforms/0068-enhanced-json-output-grouping-unit-testing.hpl diff --git a/integration-tests/transforms/0069-enhanced-json-output-grouping.hpl b/integration-tests/transforms/0068-enhanced-json-output-grouping.hpl similarity index 100% rename from integration-tests/transforms/0069-enhanced-json-output-grouping.hpl rename to integration-tests/transforms/0068-enhanced-json-output-grouping.hpl diff --git a/integration-tests/transforms/main-0068-enhanced-json-2927.hwf b/integration-tests/transforms/main-0068-enhanced-json-2927.hwf index 7fd88c5028b..30fae18c058 100644 --- a/integration-tests/transforms/main-0068-enhanced-json-2927.hwf +++ b/integration-tests/transforms/main-0068-enhanced-json-2927.hwf @@ -60,8 +60,6 @@ limitations under the License. N N ${PROJECT_HOME}/0068-enhanced-json-output-empty-stream.hpl - - Basic Y @@ -83,8 +81,8 @@ limitations under the License. N N - 640 - 48 + 736 + 224 @@ -93,8 +91,51 @@ limitations under the License. DUMMY N - 336 - 160 + 1264 + 48 + + + + 0068-enhanced-json-output-grouping.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/0068-enhanced-json-output-grouping.hpl + + + Basic + + Y + + N + direct + N + N + Y + N + 736 + 48 + + + + Run Pipeline Unit Tests + + RunPipelineTests + + + + 0068-enhanced-json-grouping-test + + + N + 1040 + 48 @@ -114,12 +155,47 @@ limitations under the License. N - 0068-enhanced-json-output-empty-stream.hpl + 0068-enhanced-json-output-grouping.hpl + Run Pipeline Unit Tests + Y + Y + N + + + Run Pipeline Unit Tests ok Y Y N + + 0068-enhanced-json-output-empty-stream.hpl + 0068-enhanced-json-output-grouping.hpl + Y + Y + N + + + 0068-enhanced-json-output-grouping.hpl + Run Pipeline Unit Tests + Y + Y + N + + + 0068-enhanced-json-output-grouping.hpl + not ok + Y + N + N + + + Run Pipeline Unit Tests + not ok + Y + N + N + diff --git a/integration-tests/transforms/main-0069-enhanced-json-grouping.hwf b/integration-tests/transforms/main-0069-enhanced-json-grouping.hwf deleted file mode 100644 index 50b44cbad2a..00000000000 --- a/integration-tests/transforms/main-0069-enhanced-json-grouping.hwf +++ /dev/null @@ -1,147 +0,0 @@ - - - - main-0069-enhanced-json-grouping - Y - - - - - - 2023/11/07 21:34:50.471 - - - 2023/11/07 21:34:50.471 - - - - - Start - - SPECIAL - - 1 - 12 - 60 - 0 - 0 - N - 0 - 1 - N - 50 - 50 - - - - 0069-enhanced-json-output-grouping.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/0069-enhanced-json-output-grouping.hpl - Basic - - Y - - N - direct - N - N - Y - N - 304 - 48 - - - - Success - - SUCCESS - - N - 752 - 48 - - - - Failed - - ABORT - - N - N - 304 - 192 - - - - Run Pipeline Unit Tests - - RunPipelineTests - - - - 0069-enhanced-json-grouping-test - - - N - 560 - 48 - - - - - - Start - 0069-enhanced-json-output-grouping.hpl - Y - Y - Y - - - 0069-enhanced-json-output-grouping.hpl - Failed - Y - N - N - - - 0069-enhanced-json-output-grouping.hpl - Run Pipeline Unit Tests - Y - Y - N - - - Run Pipeline Unit Tests - Success - Y - Y - N - - - - - - diff --git a/integration-tests/transforms/metadata/unit-test/0069-enhanced-json-grouping-test.json b/integration-tests/transforms/metadata/unit-test/0068-enhanced-json-grouping-test.json similarity index 93% rename from integration-tests/transforms/metadata/unit-test/0069-enhanced-json-grouping-test.json rename to integration-tests/transforms/metadata/unit-test/0068-enhanced-json-grouping-test.json index ec40af2a336..fb84adaad9b 100644 --- a/integration-tests/transforms/metadata/unit-test/0069-enhanced-json-grouping-test.json +++ b/integration-tests/transforms/metadata/unit-test/0068-enhanced-json-grouping-test.json @@ -61,10 +61,10 @@ "transform_name": "CSV file input" } ], - "name": "0069-enhanced-json-grouping-test", + "name": "0068-enhanced-json-grouping-test", "description": "", "persist_filename": "", "trans_test_tweaks": [], - "pipeline_filename": "./0069-enhanced-json-output-grouping-unit-testing.hpl", + "pipeline_filename": "./0068-enhanced-json-output-grouping-unit-testing.hpl", "test_type": "UNIT_TEST" } \ No newline at end of file